diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json index 4e10236c..c9fd8b0a 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json @@ -8,7 +8,7 @@ "scenario_id": "scenario_007", "variable": "head_medicare_eligible", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-09-23", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -18,6 +18,17 @@ "adjudicated_on": "2026-09-05", "adjudicator": "developer", "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + } + ], "adjudicated_verdict": { "judge_model": "claude-opus-5", "judge_failure_source": "llm_error", @@ -31,7 +42,7 @@ "scenario_id": "scenario_073", "variable": "head_medicare_eligible", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-09-23", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -41,6 +52,17 @@ "adjudicated_on": "2026-09-05", "adjudicator": "developer", "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + } + ], "adjudicated_verdict": { "judge_model": "claude-opus-5", "judge_failure_source": "llm_error", @@ -54,7 +76,7 @@ "scenario_id": "scenario_074", "variable": "head_medicare_eligible", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-10-09", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -65,7 +87,7 @@ "adjudicator": "developer", "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", "judge_reference_suspect": false, - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5", @@ -87,6 +109,13 @@ "judge_failure_subtype": "age_disability", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -95,7 +124,7 @@ "scenario_id": "scenario_079", "variable": "head_medicare_eligible", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-09-30", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -106,7 +135,7 @@ "adjudicator": "developer", "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", "judge_reference_suspect": false, - "judge_rejudged_on": "2026-09-30", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -121,6 +150,13 @@ "judge_failure_subtype": "age_disability", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-30" } ], "adjudicated_verdict": { @@ -136,7 +172,7 @@ "scenario_id": "scenario_084", "variable": "head_medicare_eligible", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-10-09", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -147,7 +183,7 @@ "adjudicator": "developer", "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", "judge_reference_suspect": false, - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -169,6 +205,13 @@ "judge_failure_subtype": "age_disability", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ], "adjudicated_verdict": { @@ -184,7 +227,7 @@ "scenario_id": "scenario_023", "variable": "snap", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-10-08", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -195,7 +238,7 @@ "adjudicator": "developer", "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04).", "judge_reference_suspect": false, - "judge_rejudged_on": "2026-10-08", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -217,6 +260,13 @@ "judge_failure_subtype": "age_disability", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ], "adjudicated_verdict": { @@ -232,7 +282,7 @@ "scenario_id": "scenario_057", "variable": "snap", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-10-09", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "taxable_income_or_deductions", "adjudicated_failure_source": "prompt_ambiguity", @@ -245,8 +295,7 @@ "judge_reference_suspect": true, "reference_verdict": "unlisted_input", "reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -269,6 +318,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ], "adjudicated_verdict": { @@ -284,7 +341,7 @@ "scenario_id": "scenario_100", "variable": "snap", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-10-09", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "taxable_income_or_deductions", "adjudicated_failure_source": "prompt_ambiguity", @@ -293,9 +350,9 @@ "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification.", + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, since fixed upstream by PolicyEngine/policyengine-us#10033, which the policyengine-us 2.41.0 references include) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification.", "judge_reference_suspect": false, - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -317,6 +374,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ], "adjudicated_verdict": { @@ -332,7 +396,7 @@ "scenario_id": "scenario_057", "variable": "ssi", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-10-09", + "judged_on_utc": "2026-10-13", "judge_failure_source": "llm_error", "judge_failure_subtype": "categorical_eligibility", "adjudicated_failure_source": "prompt_ambiguity", @@ -346,7 +410,7 @@ "reference_verdict": "unlisted_input", "reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -369,6 +433,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ], "adjudicated_verdict": { @@ -425,9 +497,9 @@ "scenario_id": "scenario_067", "variable": "ssi", "judge_model": "claude-opus-5-5", - "judged_on_utc": "2026-09-30", + "judged_on_utc": "2026-10-13", "judge_failure_source": "reference_data_issue_fixed", - "judge_failure_subtype": "age_disability", + "judge_failure_subtype": "categorical_eligibility", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", "excluded_from_scoring": true, @@ -438,7 +510,7 @@ "judge_reference_suspect": true, "reference_verdict": "unlisted_input", "reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905", - "judge_rejudged_on": "2026-09-30", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -453,6 +525,13 @@ "judge_failure_subtype": "age_disability", "judge_reference_suspect": true, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_data_issue_fixed", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": true, + "judged_on": "2026-09-30" } ], "adjudicated_verdict": { @@ -467,7 +546,7 @@ "country": "us", "scenario_id": "scenario_002", "variable": "federal_income_tax_before_refundable_credits", - "judge_model": "claude-opus-5", + "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", "judge_failure_subtype": "taxable_income_or_deductions", "adjudicated_failure_source": "prompt_ambiguity", @@ -479,7 +558,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", "reasoning": "The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", - "judged_on_utc": "2026-09-05" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-05" + } + ] }, { "country": "us", @@ -493,11 +582,11 @@ "adjudicated_on": "2026-09-22", "adjudicator": "developer", "excluded_from_scoring": true, - "judge_reference_suspect": false, + "judge_reference_suspect": true, "reference_verdict": "engine_defect", "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", "reasoning": "The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", - "judge_rejudged_on": "2026-10-08", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -519,49 +608,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_007", - "variable": "state_income_tax_before_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "state_local_rule", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "Idaho Code 63-3022P; Idaho Form 39R", - "reasoning": "The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "judge_reference_suspect": true, - "judged_on": "2026-09-29" }, { "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "judge_reference_suspect": true, - "judged_on": "2026-09-30", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ] }, @@ -582,7 +635,7 @@ "reference_basis": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say", "reasoning": "The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it.", "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", - "judge_rejudged_on": "2026-10-08", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -606,47 +659,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_042", - "variable": "state_income_tax_before_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "taxable_income_or_deductions", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "state_local_rule", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", - "reasoning": "The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.", - "judge_rejudged_on": "2026-10-08", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_engine_defect", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" }, { "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-29" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_engine_defect", + "judge_failure_source": "llm_error", "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": true, - "judged_on": "2026-09-30" + "judged_on": "2026-10-08", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -666,7 +686,7 @@ "reference_verdict": "engine_defect", "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", "reasoning": "The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", - "judge_rejudged_on": "2026-10-08", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -688,49 +708,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": true, "judged_on": "2026-09-30" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_053", - "variable": "state_income_tax_before_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "state_local_rule", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "Idaho Code 63-3022P; Idaho Form 39R", - "reasoning": "The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-08", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "judge_reference_suspect": true, - "judged_on": "2026-09-29" }, { "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", "judge_failure_subtype": "thresholds_rates", "judge_reference_suspect": true, - "judged_on": "2026-09-30", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + "judged_on": "2026-10-08" } ] }, @@ -751,7 +735,7 @@ "reference_basis": "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies", "reasoning": "The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -774,6 +758,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -793,7 +785,7 @@ "reference_verdict": "unlisted_input", "reference_basis": "taxability of survivor_benefits", "reasoning": "The frozen reference is 0.00; the alternative is 4,328.39. The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72.", - "judge_rejudged_on": "2026-09-30", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -808,6 +800,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-30" } ] }, @@ -827,7 +826,7 @@ "reference_verdict": "unlisted_input", "reference_basis": "the relationship of an adult tax dependent to the filers who claim them", "reasoning": "The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5", @@ -842,6 +841,13 @@ "judge_failure_subtype": "household_unit_or_filing_status", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -923,7 +929,7 @@ "reference_verdict": "engine_defect", "reference_basis": "7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04)", "reasoning": "The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended.", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -945,6 +951,13 @@ "judge_failure_subtype": "categorical_eligibility", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -965,7 +978,7 @@ "reference_basis": "M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21", "reasoning": "The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes.", "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -989,169 +1002,13 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_091", - "variable": "state_income_tax_before_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "taxable_income_or_deductions", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", - "reasoning": "The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-29" }, { "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": true, - "judged_on": "2026-09-30", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_099", - "variable": "state_income_tax_before_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "taxable_income_or_deductions", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076", - "reasoning": "The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.", - "judge_rejudged_on": "2026-10-10", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-29" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-30", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_100", - "variable": "federal_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "taxable_income_or_deductions", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "credit_phaseout", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", - "reasoning": "The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-22" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": true, - "judged_on": "2026-09-30", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_100", - "variable": "state_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "state_local_rule", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "credit_phaseout", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", - "reasoning": "The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "state_local_rule", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", - "judge_reference_suspect": true, - "judged_on": "2026-09-29" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "state_local_rule", - "judge_reference_suspect": true, - "judged_on": "2026-09-30", + "judged_on": "2026-10-09", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] @@ -1191,7 +1048,7 @@ "reference_basis": "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits", "reasoning": "The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out.", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1214,6 +1071,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -1223,7 +1088,7 @@ "variable": "snap", "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", - "judge_failure_subtype": "categorical_eligibility", + "judge_failure_subtype": "taxable_income_or_deductions", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "categorical_eligibility", "adjudicated_on": "2026-09-22", @@ -1234,7 +1099,7 @@ "reference_basis": "weekly_hours_worked_before_lsr", "reasoning": "The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement.", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1257,132 +1122,62 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": true, "judged_on": "2026-09-30" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_118", - "variable": "snap", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "adjudicated_failure_source": "prompt_ambiguity", - "adjudicated_failure_subtype": "taxable_income_or_deductions", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": false, - "reference_verdict": "unlisted_input", - "reference_basis": "whether the listed home mortgage interest is on the home the SNAP household occupies", - "reasoning": "The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", - "judge_rejudged_on": "2026-10-09", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": false, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": false, - "judged_on": "2026-09-29" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "taxable_income_or_deductions", - "judge_reference_suspect": false, - "judged_on": "2026-09-30" - } - ] - }, - { - "country": "us", - "scenario_id": "scenario_119", - "variable": "federal_refundable_credits", - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "credit_phaseout", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "credit_phaseout", - "adjudicated_on": "2026-09-22", - "adjudicator": "developer", - "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", - "reasoning": "The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", - "judge_rejudged_on": "2026-10-09", - "judge_previous": [ - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_engine_defect", - "judge_failure_subtype": "credit_phaseout", - "judge_reference_suspect": true, - "judged_on": "2026-09-23" - }, - { - "judge_model": "claude-opus-5-5", - "judge_failure_source": "llm_error", - "judge_failure_subtype": "credit_phaseout", - "judge_reference_suspect": true, - "judged_on": "2026-09-29", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" }, { "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", - "judge_failure_subtype": "credit_phaseout", + "judge_failure_subtype": "categorical_eligibility", "judge_reference_suspect": true, - "judged_on": "2026-09-30", + "judged_on": "2026-10-09", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, { "country": "us", - "scenario_id": "scenario_119", - "variable": "state_income_tax_before_refundable_credits", + "scenario_id": "scenario_118", + "variable": "snap", "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", "judge_failure_subtype": "taxable_income_or_deductions", - "adjudicated_failure_source": "reference_engine_defect", - "adjudicated_failure_subtype": "credit_phaseout", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", "adjudicated_on": "2026-09-22", "adjudicator": "developer", "excluded_from_scoring": true, - "judge_reference_suspect": true, - "reference_verdict": "engine_defect", - "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", - "reasoning": "The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", - "judge_rejudged_on": "2026-10-09", + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "whether the listed home mortgage interest is on the home the SNAP household occupies", + "reasoning": "The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "credit_phaseout", - "judge_reference_suspect": true, + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, "judged_on": "2026-09-23" }, { "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", - "judge_failure_subtype": "state_local_rule", - "judge_reference_suspect": true, - "judged_on": "2026-09-29", - "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" }, { "judge_model": "claude-opus-5-5", - "judge_failure_source": "reference_model_issue_fixed", - "judge_failure_subtype": "credit_phaseout", - "judge_reference_suspect": true, + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -1402,7 +1197,7 @@ "reference_verdict": "engine_defect", "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", "reasoning": "The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1424,6 +1219,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -1506,7 +1308,7 @@ "reference_verdict": "affirmed", "reference_basis": "N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32", "reasoning": "New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails.", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1528,6 +1330,13 @@ "judge_failure_subtype": "credit_phaseout", "judge_reference_suspect": true, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": true, + "judged_on": "2026-10-09" } ] }, @@ -1546,7 +1355,7 @@ "reference_verdict": "affirmed", "reference_basis": "26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12", "reasoning": "The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law.", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1562,6 +1371,13 @@ "judge_reference_suspect": true, "judged_on": "2026-09-29", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-10-09" } ] }, @@ -1571,7 +1387,7 @@ "variable": "state_income_tax_before_refundable_credits", "judge_model": "claude-opus-5-5", "judge_failure_source": "llm_error", - "judge_failure_subtype": "thresholds_rates", + "judge_failure_subtype": "state_local_rule", "adjudicated_failure_source": "llm_error", "adjudicated_failure_subtype": "state_local_rule", "adjudicated_on": "2026-09-22", @@ -1581,7 +1397,7 @@ "reference_basis": "Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601", "reasoning": "Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount.", "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1605,6 +1421,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -1623,7 +1447,8 @@ "reference_verdict": "affirmed", "reference_basis": "N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025", "reasoning": "From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule.", - "judge_rejudged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1646,6 +1471,13 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-10-09" } ] }, @@ -1665,7 +1497,7 @@ "reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C)", "reasoning": "The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published).", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1690,6 +1522,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -1709,7 +1549,7 @@ "reference_basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7", "reasoning": "The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-08", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1734,6 +1574,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": true, + "judged_on": "2026-10-08", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -1754,7 +1602,7 @@ "reference_basis": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published", "reasoning": "The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana).", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1779,6 +1627,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -1845,7 +1701,7 @@ "reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A)", "reasoning": "The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment.", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", - "judge_rejudged_on": "2026-10-09", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1870,6 +1726,14 @@ "judge_reference_suspect": true, "judged_on": "2026-09-30", "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-10-09", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" } ] }, @@ -1917,8 +1781,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", "reasoning": "The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1933,6 +1797,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -1952,8 +1823,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", "reasoning": "The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1968,6 +1839,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -1987,8 +1865,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require", "reasoning": "The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release.", - "judged_on_utc": "2026-10-08", - "judge_rejudged_on": "2026-10-08", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -1996,6 +1874,13 @@ "judge_failure_subtype": "categorical_eligibility", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ] }, @@ -2012,8 +1897,8 @@ "adjudicator": "developer", "judge_reference_suspect": false, "reasoning": "The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.", - "judged_on_utc": "2026-10-08", - "judge_rejudged_on": "2026-10-08", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2021,6 +1906,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ] }, @@ -2037,8 +1929,8 @@ "adjudicator": "developer", "judge_reference_suspect": false, "reasoning": "The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.", - "judged_on_utc": "2026-10-08", - "judge_rejudged_on": "2026-10-08", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2053,6 +1945,13 @@ "judge_failure_subtype": "state_local_rule", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_later_law", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ] }, @@ -2072,8 +1971,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set", "reasoning": "The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py).", - "judged_on_utc": "2026-10-08", - "judge_rejudged_on": "2026-10-08", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2081,6 +1980,13 @@ "judge_failure_subtype": "household_unit_or_filing_status", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ] }, @@ -2100,8 +2006,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax", "reasoning": "The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).", - "judged_on_utc": "2026-10-08", - "judge_rejudged_on": "2026-10-08", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2109,6 +2015,13 @@ "judge_failure_subtype": "state_local_rule", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-10-08" } ] }, @@ -2156,8 +2069,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set", "reasoning": "The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2165,6 +2078,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -2184,8 +2104,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax", "reasoning": "The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2193,6 +2113,13 @@ "judge_failure_subtype": "state_local_rule", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -2212,8 +2139,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax", "reasoning": "The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2221,6 +2148,13 @@ "judge_failure_subtype": "state_local_rule", "judge_reference_suspect": false, "judged_on": "2026-09-29" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -2240,8 +2174,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either", "reasoning": "The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2249,6 +2183,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -2268,8 +2209,8 @@ "reference_verdict": "unlisted_input", "reference_basis": "Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either", "reasoning": "The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py).", - "judged_on_utc": "2026-10-09", - "judge_rejudged_on": "2026-10-09", + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", "judge_previous": [ { "judge_model": "claude-opus-5-5", @@ -2277,6 +2218,13 @@ "judge_failure_subtype": "taxable_income_or_deductions", "judge_reference_suspect": false, "judged_on": "2026-09-30" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" } ] }, @@ -2296,7 +2244,17 @@ "reference_verdict": "later_law", "reference_basis": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published", "reasoning": "The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana).", - "judged_on_utc": "2026-10-09" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" + } + ] }, { "country": "us", @@ -2314,7 +2272,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own", "reasoning": "The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.", - "judged_on_utc": "2026-10-09" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" + } + ] }, { "country": "us", @@ -2332,7 +2300,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own", "reasoning": "The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.", - "judged_on_utc": "2026-10-09" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" + } + ] }, { "country": "us", @@ -2350,7 +2328,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own", "reasoning": "The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.", - "judged_on_utc": "2026-10-09" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" + } + ] }, { "country": "us", @@ -2368,7 +2356,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own", "reasoning": "The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.", - "judged_on_utc": "2026-10-09" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": false, + "judged_on": "2026-10-09" + } + ] }, { "country": "us", @@ -2386,7 +2384,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "county of residence (Indiana county income tax)", "reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state.", - "judged_on_utc": "2026-10-10" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-10-10" + } + ] }, { "country": "us", @@ -2404,7 +2412,17 @@ "reference_verdict": "unlisted_input", "reference_basis": "county of residence (Indiana county income tax)", "reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state.", - "judged_on_utc": "2026-10-10" + "judged_on_utc": "2026-10-13", + "judge_rejudged_on": "2026-10-13", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-10-10" + } + ] } ] } diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv index 44804810..4585eaf3 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv @@ -1,47 +1,48 @@ country,scenario_id,variable,model,failure_source,failure_subtype,reference_suspect,annotation -us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse even though there is no dependent child. That gave it the $32,000/$44,000 joint Social Security thresholds (taxable SS $11,542) and a joint $32,300 standard deduction. Single status yields $18,542.70 of taxable SS, an $18,150 standard deduction plus the $6,000 senior deduction, and $26,287.11 taxable, with part of it in the 12% bracket." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It computed taxable SS correctly at $18,542 but applied the $32,200 joint standard deduction for qualifying surviving spouse status, which requires a dependent child. The single deduction is $16,100 plus $2,050 aged plus $6,000 senior. So taxable income is $26,287, not $10,586, and part of it falls in the 12% bracket." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included the nontaxable $48,000 of veterans benefits in gross income and subtracted ESI premiums from AGI even though there are no wages. It then discarded its own roughly $6,600 tax estimate and asserted $0 with no supporting computation. The correct taxable income after the single standard, aged, and senior deductions is $26,287, which produces $2,906.45." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed taxable income falls below zero after deductions. In fact, AGI of $50,437 (including $18,542.70 of taxable Social Security) less the $18,150 single age-adjusted standard deduction and the $6,000 senior deduction leaves $26,287 taxable, which owes $2,906.45." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"In the taxable Social Security formula it used the full half of benefits ($13,048) instead of the lesser of that and $4,500 for single filers. That made 85% ($22,182) taxable instead of $18,542.70, so AGI came out at $54,076 instead of $50,437. It also put the 10% bracket ceiling at $12,150 rather than the 2026 figure of $12,400." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It assumed qualifying surviving spouse status without a dependent child, which inflated the deduction to about $40,000. It also overstated taxable SS at the full 85%, then declared taxable income 'near zero' even though its own figures left about $14,000. Single filing gives $26,287 of taxable income and $2,906.45 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It overstated taxable Social Security at the full 85% ($22,182) because it skipped the $4,500 single-tier cap; the correct amount is $18,542.70. It then computed about $3,300 of tax and replaced it with an unsupported $1,176. The correct path is $26,287 of taxable income, which produces $2,906.45." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It filed as a qualifying surviving spouse and used the $32,200 joint standard deduction, but that status requires a dependent child, and none is listed. Single filing uses $16,100 plus $2,050 aged plus $6,000 senior. Taxable income is therefore $26,287, not $10,586." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It used qualifying surviving spouse status (no dependent child qualifies) with a joint standard deduction of about $31,550, and it took 85% of benefits as taxable SS. It also left out the $6,000 OBBBA senior deduction. Single status gives taxable SS of $18,542.70, total deductions of $24,150, and $26,287 taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It applied qualifying surviving spouse status, using MFJ thresholds and a $33,850 deduction without a dependent child. It counted 85% of SS as taxable, omitted the $6,000 senior deduction, and then arbitrarily raised its own $2,023 result to $3,204. Single filing gives $26,287 of taxable income and $2,906.45 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It used the $32,200 joint standard deduction for surviving-spouse status, which requires a dependent child. With the correct single $18,150 age-adjusted standard deduction plus the $6,000 senior deduction, taxable income is $26,287 rather than $10,586." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its AGI of $50,436 was right, but it used an outdated $15,400 single base deduction instead of $16,100 and omitted the $6,000 OBBBA senior deduction. That left $32,986 taxable instead of $26,287. It also set the 10% bracket ceiling at $12,025 instead of $12,400." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It counted the full 85% of Social Security ($22,182) as taxable instead of $18,542.70 under the single $4,500-tier formula. It also omitted the $6,000 senior deduction and used a $17,350 standard deduction. Together these produced $36,726 of taxable income instead of $26,287." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law, with a $5,300 personal exemption and a 15% second bracket, even though OBBBA made the TCJA structure permanent. It also counted ESI premiums as a medical deduction. Under 2026 law the filer takes the $18,150 standard deduction plus the $6,000 senior deduction, and $26,287 is taxed at 10%/12%." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It got AGI ($50,436) and the $18,150 single age-adjusted standard deduction right but omitted the $6,000 OBBBA senior deduction for filers 65 and older. That left $32,286 taxable instead of $26,287. It also used a $12,200 10% bracket ceiling instead of $12,400." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, applying a personal exemption, a 15% bracket, and itemizing with ESI premiums treated as medical expenses. OBBBA kept the TCJA rate and deduction structure. The correct computation uses the $18,150 standard deduction plus the $6,000 senior deduction and the 10%/12% brackets on $26,287." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $37,990 taxable income leaves out the $6,000 senior deduction and understates the age-adjusted standard deduction, and its $1,109 tax does not follow from that taxable income under any bracket schedule. AGI of $50,437 less $24,150 of deductions gives $26,287 taxable and $2,906.45 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied the expired pre-TCJA regime: a roughly $5,400 personal exemption, itemized deductions, and a 15% bracket. OBBBA made TCJA permanent, so there is no personal exemption, the filer takes the $18,150 standard deduction plus the $6,000 senior deduction, and the brackets are 10%/12%." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA reverted in 2026 (a $5,050 personal exemption, itemizing $16,470, and a 15% bracket above $11,000). Under OBBBA-permanent law the $18,150 standard deduction plus the $6,000 senior deduction exceed itemizing. That gives $26,287 of taxable income at 10%/12%." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that income falls below the standard deduction without computing taxable Social Security. AGI is $50,437 including $18,542.70 of taxable SS, and after $24,150 of standard, aged, and senior deductions, $26,287 remains taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used the expired pre-TCJA structure: itemized property taxes plus a personal exemption, taxed in 10%/15% brackets. Under 2026 OBBBA law there is no personal exemption, and the $18,150 standard deduction plus the $6,000 senior deduction yield $26,287 of taxable income." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It itemized property taxes, took a personal exemption, and applied the pre-TCJA 15% bracket. OBBBA made TCJA permanent, so the single filer takes the $18,150 standard deduction plus the $6,000 senior deduction and pays 10%/12% on $26,287." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $50,436 was correct, but it used a $17,000 standard deduction instead of $18,150 ($16,100 plus $2,050 aged) and omitted the $6,000 senior deduction. That left $33,436 taxable instead of $26,287." -us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It correctly filed as single with $50,436 AGI but omitted the $6,000 OBBBA senior deduction and used a $17,510 estimate instead of the $18,150 age-adjusted standard deduction. That left $32,926 taxable instead of $26,287." -us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $6,000 senior deduction, which applies whether or not the filer itemizes. It also treated the $8,389 ESI premiums as a deductible medical expense, which pushed itemized deductions to $21,076. The correct itemized total is $17,587.20, so the $18,150 standard deduction wins and taxable income is $26,287." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It relied on a 'surviving-spouse rate structure' that does not apply without a dependent child and claimed the deductions fully offset income. Single-status AGI of $50,437 less $24,150 of deductions leaves $26,287 taxable and $2,906.45 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that the age-65 standard deduction wipes out all income, ignoring that $18,542.70 of Social Security is taxable and that AGI reaches $50,437. The total deductions of $24,150 leave $26,287 taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint thresholds, so taxable SS came out at $11,542, along with a joint $32,200 standard deduction, but the head has no dependent child and files single. Single thresholds make $18,542.70 of SS taxable, and the $24,150 of deductions leave $26,287 taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the right single-filer structure but miscalculated taxable Social Security as $18,592 instead of $18,542.70. That pushed AGI to $50,486 and taxable income to $26,336 instead of $26,287.11, about $49 too high, so it overshot the tax." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It filed as a surviving spouse with joint thresholds and a joint standard deduction even though there is no dependent child. That left only about $3,590 taxable instead of the $26,287 that results under single status." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It applied a $33,850 surviving-spouse (joint) age-adjusted standard deduction, but without a dependent child the head files single. The correct deductions are $18,150 plus the $6,000 senior deduction, giving $26,287 of taxable income rather than $10,586." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It followed the correct single path but overstated taxable Social Security by about $49. That carried through to taxable income of $26,336 instead of $26,287.11, so it reported $2,912 instead of $2,906.45." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint SS thresholds (taxable SS $11,542) and a $33,850 joint deduction, but qualifying surviving spouse status requires a dependent child. Single status gives $18,542.70 of taxable SS and $26,287 of taxable income." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard and senior deductions exceed taxable income. In fact AGI is $50,437 and total deductions are $24,150 ($16,100 + $2,050 + $6,000), which leaves $26,287 taxable." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law (a $5,300 personal exemption, a reverted $16,600 standard deduction, and a 15% bracket) and used joint bracket widths for surviving-spouse status without a dependent child. Under 2026 OBBBA single rules, the $18,150 standard deduction plus the $6,000 senior deduction give $26,287 taxed at 10% up to $12,400 and 12% above." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA expired, restoring the personal exemption and the 15% bracket, and itemized property taxes. Under OBBBA-permanent 2026 law the $18,150 standard deduction plus the $6,000 senior deduction apply, giving $26,287 taxed at 10%/12%." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its filing status, AGI, and deductions were all correct ($26,286 taxable), but it put the 2026 single 10% bracket ceiling at $12,200 instead of $12,400. That shifted $200 from the 10% to the 12% rate and added about $4 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA sunset (uncapped SALT and a 15% bracket) and treated ESI premiums as an itemizable medical expense. It also omitted the $6,000 senior deduction. Under 2026 OBBBA law the $18,150 standard deduction plus the $6,000 senior deduction give $26,287 taxed at 10%/12%." -us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output. The correct derivation is single status, $50,437 of AGI, $24,150 of deductions, and $26,287 taxable, which gives $2,906.45." -us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It subtracted a $750 elderly credit and invented arithmetic to reach $0. The elderly credit is fully phased out because AGI of $50,437 far exceeds the $7,500 single threshold, and no other credit applies, so the tax on $26,287 of taxable income stands at $2,906.45." -us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It left half of Social Security out of provisional income (using $37,472 instead of $50,520) and dropped the $4,500 first-tier amount. That made only $2,951 of SS taxable instead of $18,542.70 and understated AGI by about $15,600. Its single filing status and $24,150 of deductions were correct." -us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It filed as a qualifying widow(er) with a joint $31,150 deduction although there is no dependent child. It also declared Social Security nontaxable even though provisional income of $50,520 exceeds every base amount. Single status makes $18,542.70 of SS taxable and leaves $26,287 taxable after the $24,150 of deductions." -us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the single 65+ standard deduction eliminates taxable income. AGI is $50,437 including taxable Social Security, and the $18,150 standard deduction plus the $6,000 senior deduction leave $26,287 taxable, which owes $2,906.45." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Treated the head as a qualifying surviving spouse, but that status requires a dependent child, so the filer is single. Using the joint $32,000/$44,000 Social Security thresholds understated taxable SS ($11,542 instead of $18,542.70), and the joint $32,300 standard deduction replaced the single $16,100. This cut taxable income to $3,486 when it should be $26,287.11." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Taxable SS of $18,542 and AGI were correct, but the model applied the joint/QSS $32,200 standard deduction, $1,650 aged add-on and joint brackets. With no dependent child the head files single, so the deductions are $16,100 + $2,050 + $6,000 = $24,150, leaving $26,287.11 of taxable income that reaches the 12% bracket." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Put the nontaxable $48,000 of veterans benefits into gross income and subtracted ESI premiums even though there were no wages. It then dropped its own roughly $6,600 estimate and answered $0 with no computation. The correct single-filer computation leaves $26,287.11 of taxable income and $2,906.45 of tax." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Said taxable income was below the tax threshold. In fact, AGI of $50,437.11 minus the $18,150 aged single standard deduction and the $6,000 senior deduction leaves $26,287.11 taxable, which produces $2,906.45 of tax." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Filing status and deductions were correct, but the Social Security formula used half of the benefits ($13,048) as the base instead of the lesser of $4,500 or half the benefits. This pushed taxable SS to the $22,182 cap when the correct figure is $18,542.70. The model also ended the 10% bracket at $12,150 instead of $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Assumed qualifying surviving spouse (joint) status even though no dependent child is listed, and overstated taxable SS at $22,182. It then claimed taxable income was 'near zero', which contradicts its own roughly $14,000 residual. The correct single computation leaves $26,287.11 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Set taxable SS at the 85% cap of $22,182 instead of $4,500 + 85% × ($50,520 − $34,000) = $18,542.70. It then cut its own roughly $3,300 tax estimate to $1,176 with no computation behind the change." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"Used the joint/surviving-spouse $32,200 standard deduction and $1,650 aged add-on even though a widowed filer with no dependent child is single. The single deduction totals $16,100 + $2,050 + $6,000, leaving $26,287.11 of taxable income instead of $10,586." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Treated the head as a qualifying surviving spouse and used a joint standard deduction, even though there is no dependent child and the filer is single. It also set taxable SS at the full 85% ($22,182), which its own $32,000/$44,000 thresholds don't support, and it left out the $6,000 senior deduction entirely." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Used QSS/joint status even though there is no dependent child, set taxable SS at $22,182 instead of $18,542.70, and left out the $6,000 senior deduction. It then raised its own $2,023 computation to $3,204 with an unexplained 'adjustment'." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"Applied the joint/surviving-spouse $32,200 standard deduction and $1,650 aged add-on. A widowed filer with no dependent child files single, so the deduction is $18,150 (plus the $6,000 senior deduction), leaving $26,287.11 of taxable income instead of $10,586." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Left out the $6,000 OBBBA senior deduction and used an outdated $15,400 single standard deduction instead of $16,100. It also ended the 10% bracket at $12,025 instead of $12,400. These errors produced $32,986 of taxable income when the correct figure is $26,287.11." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Overstated taxable SS at the $22,182 cap when the single-threshold formula gives $18,542.70. It also used a $17,350 standard deduction instead of $18,150 and left out the $6,000 senior deduction. Together these produced $36,726 of taxable income instead of $26,287.11." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied pre-TCJA sunset law, with a $5,300 personal exemption and a 15% second bracket, but OBBBA made the TCJA structure permanent for 2026. It also deducted ESI premiums as a medical expense. The correct computation uses the $18,150 standard deduction plus the $6,000 senior deduction with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Used the correct $18,150 single aged standard deduction but left out the $6,000 OBBBA senior deduction for filers 65 and older. It also ended the 10% bracket at $12,200 instead of $12,400, which leaves $32,286 taxable instead of $26,287.11." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed TCJA expired in 2026, so it used a personal exemption, an uncapped itemized deduction with ESI premiums counted as medical expenses, and a 15% bracket. Under OBBBA the filer takes the $18,150 standard deduction plus the $6,000 senior deduction and is taxed at 10%/12%." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Stated taxable income of $37,990, which leaves out the $6,000 senior deduction and part of the standard deduction, and the $1,109 answer does not follow from applying any bracket schedule to that figure. The correct taxable income is $26,287.11, producing $2,906.45 of tax." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Assumed TCJA sunset rules, with a personal exemption, itemized real estate taxes and 10%/15% brackets. Under OBBBA the filer takes the $18,150 standard deduction plus the $6,000 senior deduction with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied pre-TCJA law for 2026, with a $5,050 personal exemption, itemized deductions and a 15% bracket above $11,000, even though OBBBA kept the TCJA standard deduction and rates. The correct deductions are $18,150 + $6,000, taxed at 10% up to $12,400 and 12% above." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed income was below the standard deduction and exemptions. In fact, AGI of $50,437.11 exceeds the $24,150 of total deductions by $26,287.11, which is taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied TCJA-sunset rules, with itemized real estate taxes plus a personal exemption and 15% brackets. Under OBBBA 2026 law the filer takes the $18,150 standard deduction plus the $6,000 senior deduction under 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used pre-TCJA law (a personal exemption, itemized property tax and a 15% bracket), even though OBBBA makes the $16,100 + $2,050 standard deduction, the $6,000 senior deduction and the 10%/12% brackets apply in 2026." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Left out the $6,000 OBBBA senior deduction and used a $17,000 aged single standard deduction instead of $18,150. This produced $33,436 of taxable income instead of $26,287.11." +us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Left out the $6,000 OBBBA senior deduction. It also estimated the single aged standard deduction at $17,510 instead of $18,150 and ended the 10% bracket at $12,283 instead of $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Itemized $21,076 by counting the ESI premiums as a medical expense, even though they are not itemizable medical costs, and left out the $6,000 senior deduction. The correct deductions are the $18,150 standard deduction plus $6,000, leaving $26,287.11 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"Applied a surviving-spouse rate structure even though there is no dependent child, and claimed all income was offset. The single computation leaves $26,287.11 of taxable income and $2,906.45 of tax." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Said the standard deduction fully offsets income, but AGI of $50,437.11 exceeds the $18,150 standard deduction plus the $6,000 senior deduction by $26,287.11, which is taxed at 10%/12%." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Used the joint/surviving-spouse $32,000/$44,000 Social Security thresholds and the $32,200 joint standard deduction, even though the head has no dependent child and files single. This understated taxable SS ($11,542 instead of $18,542.70) and overstated the standard deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Filing status, deductions and brackets were correct, but taxable SS was miscomputed as about $18,592 instead of $4,500 + 0.85 × $16,520 = $18,542.70. This overstated taxable income by about $49 and tax by about $6." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Filed as a surviving spouse with joint thresholds and standard deduction even though no dependent child exists. That left only about $3,590 taxable at 10%, when single status gives $26,287.11 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Applied the $33,850 joint/surviving-spouse aged standard deduction even though the head has no dependent child and must file single. The correct deduction is $18,150, which leaves $26,287.11 taxable instead of $10,586." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct single deductions and brackets but overstated taxable SS by about $49 (about $18,592 instead of $18,542.70). That produced $26,336 of taxable income instead of $26,287.11." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"Used joint/surviving-spouse Social Security thresholds and the $33,850 joint aged standard deduction, even though a widowed filer with no dependent child is single. This understated taxable SS ($11,542 instead of $18,542.70) and left only $3,586 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Asserted that the senior deductions exceed taxable income. The $18,150 standard deduction plus the $6,000 senior deduction leave $26,287.11 of AGI taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Applied TCJA-sunset law (a $16,600 standard deduction plus a $5,300 personal exemption and a 15% bracket) and used a joint-width $23,200 10% bracket for a QSS filer with no dependent child. Under OBBBA single rules the deductions are $16,100 + $2,050 + $6,000, and the 10% bracket ends at $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Assumed restored pre-TCJA law, with a personal exemption, itemized property tax and 10%/15% brackets. Under OBBBA 2026 the filer takes the $18,150 standard deduction plus the $6,000 senior deduction with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Got taxable income right at $26,286 but ended the 2026 single 10% bracket at $12,200 instead of $12,400. Taxing that extra $200 at 12% instead of 10% overstated the tax by about $4." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Applied TCJA-sunset law (no SALT cap, a 15% bracket), counted ESI premiums as a medical deduction, and left out the $6,000 senior deduction. Under OBBBA the filer takes the $18,150 standard deduction plus $6,000 and is taxed at 10%/12%." +us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value for federal_income_tax_before_refundable_credits, so the required output is missing." +us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Subtracted credits that don't apply: a Lifetime Learning credit with no education expenses, and a $750 elderly credit that the high nontaxable Social Security and AGI eliminate. That drove tax to $0. The correct computation leaves $2,906.45 with no nonrefundable credits." +us,scenario_000,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Left out the $6,000 OBBBA senior deduction for filers 65 and older, which leaves $32,286 taxable instead of $26,287.11. Everything else was correct, so the omission alone overstates tax by $720." +us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Left half of the Social Security benefits out of provisional income ($37,472 instead of $50,520) and dropped the $4,500 base-amount tier. This produced $2,951 of taxable SS instead of $18,542.70." +us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"Treated the full $26,096 of Social Security as nontaxable even though provisional income of $50,520 exceeds the $34,000 threshold. It also used a $31,150 QSS standard deduction for a filer with no dependent child, who files single." +us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the aged single standard deduction wipes out taxable income. AGI of $50,437.11 minus the $18,150 standard deduction and the $6,000 senior deduction still leaves $26,287.11 taxable." us,scenario_000,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_000,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age 77 and an asserted income exclusion calculation as sufficient for Texas aged Medicaid without identifying or satisfying an aged eligibility pathway. The person qualifies through no Medicaid category and receives no SSI, so the claimed below-threshold aged eligibility does not apply." us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no earned wages were reported and that payroll tax therefore equals zero, but then submitted $1,534 instead. Its numeric output contradicts its own payroll-tax-base analysis and the required final-value contract." @@ -73,16 +74,17 @@ us,scenario_001,state_income_tax_before_refundable_credits,glm-5.2,llm_error,sta us,scenario_001,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $250.20 implies about $7,600 of Virginia taxable income, i.e. roughly $21,460 of total deductions against $29,064 of Virginia AGI — consistent with a single $12,000 age deduction plus the standard deduction and exemptions, not the ""two age deductions"" its explanation claims. Both spouses are 65 or older and combined AFYA of $29,064 is below the $75,000 joint phase-out, so $24,000 of age deductions applies and by itself exceeds Virginia AGI, leaving $0 of taxable income and $0 of tax." us,scenario_001,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated Virginia's age benefit as the $1,600 of ""age/blind additions"" — the two $800 additional 65+ exemptions — and never claimed the separate $12,000-per-person age deduction of § 58.1-322.02(10)-(11), which is unreduced because AFYA of $29,064 is far under the $75,000 joint threshold. Adding the missing $24,000 to its own $16,000 standard deduction and $1,860 of exemptions eliminates the $9,600 of taxable income it computed, so the tax is $0 rather than $350." us,scenario_001,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Put the entire $15,225 of Social Security into federal AGI for a $44,289 starting point when only $2,338.25 is federally taxable under the 50% tier, and then never applied Virginia's subtraction for that taxable portion — Virginia AGI is $29,064. It also discarded its own branch that used the correct $24,000 of age deductions and submitted $739.15, a figure none of its three calculations ($449.37, $33.78, and the $10,989 taxable-income branch) produces. Virginia AGI of $29,064 less $24,000 of age deductions plus the standard deduction and exemptions yields $0 taxable income and $0 tax." -us,scenario_002,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It computed the return as a single filer — stating ""only the head's income is relevant"" — applying the single-filer $25,000/$34,000 provisional-income thresholds and a $20,550 single 65+ standard deduction instead of joint treatment, and it also carried the $24,000 disability benefits into both provisional income and includible income after asserting they were excluded. On the joint return with the disability and veterans benefits excluded, provisional income is $40,780, taxable Social Security is 50% × ($40,780 − $32,000) = $4,390, and AGI of $23,130 falls below the $35,500 MFJ standard deduction ($32,200 + two $1,650 age additions), before the $12,000 senior deduction, so tax is $0 rather than $4,427." -us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $24,000 disability benefits as taxable, which raised provisional income to $64,780, past the $44,000 MFJ threshold, and produced $23,663 of taxable Social Security instead of the $4,390 that the 50%-tier calculation yields at the true $40,780 provisional income. It then subtracted a pre-TCJA deduction stack — a $15,650 standard deduction plus two $5,050 personal exemptions, which have been repealed since 2018 — and applied a 15% bracket that no longer exists, when the actual 2026 MFJ deductions ($32,200 + $3,300 age additions + $12,000 senior deduction) exceed the $23,130 AGI and zero out the tax." -us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reached the correct conclusion — the 2026 married-both-65+ deduction exceeds AGI, so tax before refundable credits is $0 — and its explanation ends ""value = 0,"" but the submitted numeric value was $120.30. The scored answer diverges from the model's own stated derivation, so this is a contract violation between explanation and submitted value rather than a substantive tax error." -us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $24,000 disability benefits as taxable, driving provisional income above the $44,000 MFJ threshold and forcing the 85%-tier Social Security inclusion of $23,663 rather than the 50%-tier $4,390 that $40,780 of provisional income produces. It then reduced the resulting $66,403 by $29,100 of ""standard deduction, personal exemptions, and senior additions"" — personal exemptions are repealed — and applied a nonexistent 15% bracket; with the disability and veterans benefits excluded, AGI of $23,130 is below the $35,500 MFJ standard deduction and the tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It included the full $44,080 of Social Security in income with no provisional-income computation, when the §86 test caps inclusion at 50% of the $8,780 excess over the $32,000 MFJ threshold, or $4,390. That single error produced $62,820 of income against a $34,500 deduction; the correct AGI of $23,130 is below the $35,500 MFJ standard deduction (and further below it after the $12,000 senior deduction), so tax is $0 instead of $3,122." -us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $22,203 of taxable income is exactly $66,403 less $44,200, meaning it correctly applied the 2026 MFJ $32,200 standard deduction plus the $12,000 senior deduction but built AGI by adding the $24,000 of non-taxable disability benefits, which also pushed provisional income to $64,780 and swapped the $4,390 50%-tier Social Security inclusion for the $23,663 85%-tier amount. Excluding the disability benefits leaves AGI of $23,130, below even the base standard deduction, so tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation states that taxable income ""reflects taxable pension, interest, disability income, taxable Social Security"" — treating the $24,000 disability benefits as taxable, which also lifts provisional income to $64,780 and adds $19,273 of extra Social Security inclusion ($23,663 instead of $4,390). The $3,429 result is consistent with an AGI near $66,400 reduced by roughly $33,700 of deductions; with the disability and veterans benefits excluded, AGI is $23,130, under the $35,500 MFJ standard deduction, and the tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It made two compounding errors: it put the $24,000 of non-taxable disability benefits into provisional income ($64,780 versus the correct $40,780), and then took a flat 85% of gross benefits ($37,468) instead of the §86 lesser-of test, which even at its own $64,780 caps inclusion at 0.85 × ($64,780 − $44,000) + $6,000 = $23,663. Correctly, only $4,390 of Social Security is includible, AGI is $23,130, and that is below the $35,500 MFJ standard deduction it approximated as $34,110, before the $12,000 senior deduction, so tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $24,000 disability benefits to taxable income, which is the sole driver of its $66,403 AGI and which also raised provisional income past $44,000 and produced $23,663 of taxable Social Security rather than the 50%-tier $4,390. Its $35,500 deduction figure ($32,200 base plus $3,300 of age additions) is the correct base amount and already exceeds the true $23,130 AGI, so removing the disability benefits alone drives taxable income and tax to $0." -us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"After correctly dropping the disability and veterans benefits, it declared the full $44,080 of Social Security ""fully taxable since combined income exceeds the threshold"" — §86 never includes more than 85%, and at a provisional income of $40,780 (between the $32,000 and $44,000 MFJ thresholds) the includible amount is 50% × $8,780 = $4,390. Substituting that gives AGI of $23,130, below its own $32,600 deduction estimate and far below the actual $35,500 standard deduction plus $12,000 senior deduction, so tax is $0 instead of $3,149.40." +us,scenario_002,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"In its final recalculation the model added the $24,000 disability benefits back into income ($42,740 other income). It applied the single-filer Social Security thresholds of $25,000/$34,000 instead of the MFJ $32,000/$44,000, and used a single $20,550 deduction and single brackets instead of the joint $35,500 deduction plus $12,000 senior deduction. With disability excluded and MFJ rules applied, AGI is $23,130, which is less than the deductions, so tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"The model counted the $24,000 disability benefits as taxable, which raised other income and pushed taxable Social Security to $23,663 instead of $4,390. It also used a $15,650 single-style deduction plus personal exemptions, which are suspended, and a 15% bracket that no longer exists. It never applied the MFJ 65+ deduction of $35,500 plus the $12,000 senior deduction, which would bring the correct $23,130 AGI to $0 tax." +us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"The model's reasoning concludes that the MFJ 65+ standard deduction exceeds AGI and ends with 'value = 0', but the submitted numeric value is $120.30. That number contradicts its own explanation, so the error is in the output, not the tax analysis." +us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the $24,000 disability benefits as taxable, which produced an AGI of $66,403 with $23,663 of taxable Social Security. The correct AGI is $23,130 with only $4,390 of Social Security taxable. It also used personal exemptions, which are suspended, and a 15% bracket instead of the $35,500 MFJ 65+ deduction plus the $12,000 senior deduction." +us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model correctly left out the disability benefits but counted the full $44,080 of Social Security as taxable instead of running the provisional-income test. That test gives provisional income of $40,780, which is under $44,000, so only 50% × ($40,780 − $32,000) = $4,390 is taxable. It also left out the $12,000 senior deduction. The correct AGI of $23,130 is below the deductions, so tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"The model included the $24,000 disability benefits in taxable income, which inflated the Social Security inclusion and gave a positive taxable income of about $22,203. PolicyEngine excludes disability benefits from gross income, so AGI is only $23,130, the joint standard, age, and senior deductions reduce taxable income to zero, and tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"The model counted 'disability income' as taxable alongside the pension, interest, and Social Security, which gave a large positive taxable income. With the $24,000 disability benefits excluded, taxable Social Security is $4,390 and AGI is $23,130, below the $35,500 MFJ 65+ standard deduction alone, so tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model included the $24,000 disability benefits in provisional income, and then treated the full 85% cap ($37,468) as taxable Social Security. Even under its own figures, the formula gives min($37,468, 85% × ($64,780 − $44,000) + $6,000 = $23,663). It also left out the $12,000 senior deduction. With disability excluded, taxable Social Security is $4,390, AGI is $23,130, and tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the $24,000 disability benefits as taxable, which gave an AGI of $66,403 instead of $23,130. It also deducted only $35,500 and left out the $12,000 senior deduction ($6,000 per spouse 65+). Without the disability benefits, AGI is below the standard deduction and tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the deductions correctly, totaling $47,500, but counted the $24,000 disability benefits as taxable. That raised AGI to $66,403 and taxable Social Security to $23,663. PolicyEngine excludes disability benefits, so AGI is $23,130 with only $4,390 of Social Security taxable, and taxable income and tax are both $0." +us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model excluded the disability benefits in its final pass but declared the full $44,080 of Social Security taxable. Its own combined income of $40,780 falls below the $44,000 MFJ second threshold, so only 50% × ($40,780 − $32,000) = $4,390 is taxable. It also used a $32,600 deduction that leaves out the $12,000 senior deduction. The correct AGI of $23,130 gives $0 tax." us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified a zero taxable wage base and then contradicted that computation by inventing taxable wage-equivalent income and assigning $1,926 of Social Security and Medicare taxes. No listed income is employee compensation subject to payroll tax, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components all equal zero." us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly evaluated the spouse as a separate zero-income individual and assumed that age alone opened an aged Medicaid pathway. Washington's pathway evaluation includes the applicable household income and categorical requirements; this spouse receives no SSI, has household MAGI of 2.90 times FPL, and qualifies for no category." us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working reached about $22,379 using a $32,300 standard deduction and the correct 2026 brackets. It then submitted $27,049, a number that nothing in its reasoning supports." @@ -148,258 +150,264 @@ us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 implies about $64,000 of New York taxable income under the MFJ brackets — more than the household's entire $52,540 of gross receipts including Social Security — so it both taxed the $51,475 benefit and imputed unlisted wage earnings from the 40-hour work weeks that the prompt requires be treated as 0. The actual New York base is $1,065 of interest less partnership loss, fully offset by the ~$16,050 MFJ standard deduction, for $0 of tax." us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrote that New York does not tax Social Security yet built its base from $52,540 with the full $51,475 benefit included, never applying the federal provisional-income test ($26,803 against the $32,000 MFJ base) that makes the benefit 0% taxable and therefore leaves only $1,065 of federal AGI to flow to New York. Its arithmetic is also self-contradictory: $52,540 less its stated $16,850 standard deduction is $35,690, not the $18,426 it claimed, and $3,451.12 on $18,426 is an 18.7% effective rate against NY's 4%–4.5% brackets at that level (about $743). The correct result is $1,065 of NY AGI wiped out by the ~$16,050 standard deduction, giving $0." us,scenario_004,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model correctly found federal AGI under $5,000 and used the table amount of $90 plus $15 for the second exemption, but it counted the NY household credit (Tax Law §606(b)) as refundable. That credit is nonrefundable: it only offsets NY tax, which is zero here, and it never enters state_refundable_credits. None of NY's refundable credits apply (EITC, Empire State child credit, real property tax credit), so the answer is $0, not $105." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Left out the 3.8% net investment income tax ($5,721.98 on $150,578.34 of NII), which this output includes; its ~$100.2k regular-tax figure was nudged to $100,819 and NIIT was never added. It also kept the full $46,308 of 401(k) exclusions instead of the $41,650 left after pro-rating to the $24,500 combined traditional+Roth limit, and it denied the $2,164 IRA deduction and the $1,070 non-itemizer charitable deduction." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state tax refund as income (PolicyEngine excludes it when no prior-year itemization benefit exists), excluded the full $46,308 of 401(k) deferrals instead of $41,650 pro-rated to the $24,500 combined traditional+Roth limit, and left out the $1,070 OBBBA non-itemizer cash charitable deduction. The errors partly offset, leaving taxable income of $504,054 versus $503,494.75 and tax $178.94 too high at 32%." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $110,000 of mortgage interest from the $2.85M balance even though no interest paid was listed, so mortgage interest is $0 and the $32,200 standard deduction plus $1,070 non-itemizer charity applies. Its $249,914 also does not follow from its own ~$408,000 taxable income; the correct path is $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Refused to compute the liability and entered a placeholder $0 despite $503,494.75 of taxable income. Ordinary bracket tax ($89,095.92), 15% tax on $77,920 of qualified dividends ($11,688), and NIIT ($5,721.98) add up to $106,505.90." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly excluded NIIT ($5,721.98) from this line, then reported $91,458, which also drops its own $11,688 qualified-dividend tax and roughly equals its ordinary-bracket tax alone. It also used outdated 2026 brackets (24% ending near $396,000 instead of $403,550) and the full $46,308 401(k) exclusion instead of the pro-rated $41,650." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Invented mortgage interest from the $2.85M balance even though no interest paid was listed, producing ~$60,000 of itemized deductions instead of the $32,200 standard deduction plus $1,070 non-itemizer charity. It also taxed qualified dividends at 20% even though ~$503k of taxable income falls in the 15% band, and its $154,430 exceeds the sum of its own components (~$92,000 + $15,584) by about $47,000." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Added AGI up to $604,318 when its own listed components sum to $538,418, and invented mortgage interest from the $2.85M balance (no interest paid was listed) to reach ~$60,000 of itemized deductions. It also used an 18.8% blended rate on qualified dividends instead of 15%, overshooting the correct $89,095.92 + $11,688 + $5,721.98." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state tax refund as gross income (PolicyEngine excludes it), excluded the full $46,308 of 401(k) deferrals instead of the $41,650 pro-rated to the $24,500 combined traditional+Roth limit, and left out the $1,070 OBBBA non-itemizer cash charitable deduction. Its taxable income of $504,054 versus $503,494.75 overstated tax by $178.94 at 32%." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset law for 2026: pre-TCJA 15/25/28/33% brackets, restored 2%-floor miscellaneous deductions, and a pre-TCJA AMT exemption that produced $7,141 of AMT. OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $140,200 MFJ AMT exemption (phase-out starting at $1,000,000) permanent. It also invented $52,500 of mortgage interest from a balance with no stated interest and computed NIIT on $153,578 without the $3,000 capital-loss offset." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Invented ~$28,000 of mortgage interest from the $2.85M balance even though no interest paid was listed, and itemized with a $10,000 SALT cap instead of taking the $32,200 standard deduction plus $1,070 non-itemizer charity. It never ran the brackets and guessed $115,000; even its own $497,844 taxable income yields about $104,700 including NIIT under 2026 brackets." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Itemized the full $2.6k of charity on top of a ~$31.9k SALT deduction, ignoring the 0.5%-of-AGI floor that eliminates the itemized charitable deduction, so the $32,200 standard deduction plus $1,070 non-itemizer cash deduction is the right choice. It also counted the $4,148 refund, denied the $2,164 IRA deduction, and excluded the full $46,308 of 401(k) deferrals instead of $41,650; the offsetting errors left it $144 high." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used a $30,750 standard deduction instead of the 2026 OBBBA $32,200 plus the $1,070 non-itemizer cash charitable deduction, and got $92,234 of ordinary tax on $429,748 where 2026 brackets give $90,431. It also computed NIIT on $153,578 without netting the $3,000 allowable capital loss (correct NII is $150,578.34), counted the $4,148 refund, and used the unprorated $46,308 401(k) exclusion." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Added an AMT that does not bind: under OBBBA the 2026 MFJ AMT exemption is $140,200, phasing out only above $1,000,000 of AMTI, so tentative minimum tax on ~$537k of AMTI stays below the $100,783.92 regular tax. Its $465,449 taxable income also reflects itemized deductions far above the $33,270 standard-plus-charity deduction, and its $5,836 NIIT ignores the $3,000 capital-loss offset." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules, namely the Pease limit on itemized deductions and an AMT that raised tax to $127,938, but OBBBA permanently repealed Pease and set the 2026 AMT exemption at $140,200 with a $1,000,000 phase-out threshold, so no AMT arises. It also claimed ~$59.5k of itemized deductions with no listed mortgage interest instead of the $32,200 standard deduction plus $1,070 non-itemizer charity, and used $153,578 of NII without the $3,000 capital-loss offset." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Taxed ordinary income on pre-TCJA 'sunset' 2026 brackets and itemized $63,507.64, but OBBBA made the TCJA brackets permanent, and with no mortgage interest listed the household takes the $32,200 standard deduction plus $1,070 non-itemizer charity. It also computed NIIT on $153,578 instead of $150,578.34 after the $3,000 capital loss." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expired, using uncapped SALT, mortgage interest invented on $1M of acquisition debt with no interest listed, and $10,400 of personal exemptions, for $108,283 of itemized deductions. Under OBBBA personal exemptions stay at zero, SALT is capped at a phased-down ~$30.9k, and the $32,200 standard deduction plus $1,070 non-itemizer charity applies. Its errors netted to $88,016 of ordinary tax, $1,080 below the correct $89,095.92." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Reported AGI of $460,936 'including pensions', but the $70,000 pension is tax-exempt and correct AGI is $536,764.50 (wages net of the pro-rated $41,650 of 401(k) deferrals, plus interest and dividends, less the $3,000 capital loss and $2,164 IRA deduction). It gave no bracket, qualified-dividend, or NIIT breakdown, and its $110,996 overshoots by $4,490." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Added an AMT that does not arise under OBBBA's 2026 exemption ($140,200 MFJ, phase-out starting at $1,000,000 of AMTI). It also reduced wages by the $13,369 of employee ESI premiums to reach AGI of $525,049, while PolicyEngine reduces wages only by the $41,650 of pro-rated 401(k) deferrals (AGI $536,764.50)." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Treated TCJA as expired in 2026, deducting uncapped SALT (state income plus real estate tax) and using brackets that top out at 33%. OBBBA permanently kept TCJA rates and caps SALT at a phased-down ~$30.9k, which loses to the $32,200 standard deduction plus $1,070 non-itemizer charity, so the answer is $89,095.92 + $11,688 + $5,721.98." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. The correct path is taxable income of $503,494.75 ($32,200 standard deduction plus $1,070 non-itemizer charity), $89,095.92 ordinary tax, $11,688 of 15% qualified-dividend tax, and $5,721.98 NIIT. Its $106,093 is $412.90 short, which equals understating taxable income by about $1,290 at the 32% marginal rate." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (personal exemptions plus itemized deductions), subtracted the $13,369 of employee ESI premiums from wages to reach AGI of $525,049 instead of $536,764.50, and taxed qualified dividends at 20% even though ~$503k of taxable income falls in the 15% band. The correct total is $89,095.92 ordinary tax + $11,688 + $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration brackets, treated health premiums as pre-tax wage reductions, and invented mortgage interest from the $2.85M balance even though no interest paid was listed, pushing tax down to $88,392. Under OBBBA's permanent rules the household takes the $32,200 standard deduction plus $1,070 non-itemizer charity, giving $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA sunset rules, deducting uncapped state income tax and 2%-floor miscellaneous deductions for the $13,848 of unreimbursed employee expenses. Under OBBBA miscellaneous itemized deductions stay suspended, SALT is capped (phased down to ~$30.9k), and the $32,200 standard deduction plus $1,070 non-itemizer charity wins. It also used AGI of $534,270 (no IRA deduction, unprorated 401(k)) and rounded to $105,000." -us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $5,721.98 net investment income tax, which this output includes. It also used 2025 parameters (the $30,000 standard deduction and 2025 brackets) instead of 2026's $32,200 standard deduction, the $1,070 non-itemizer cash charitable deduction, and inflation-adjusted brackets." -us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Took the maximum $2,000 non-itemizer charitable deduction even though only $1,070 of cash gifts qualify (non-cash gifts are excluded), and excluded the full $46,308 of 401(k) deferrals instead of $41,650 pro-rated to the $24,500 combined limit. Its taxable income was $500,070 versus $503,494.75, and its NIIT was computed on $153,578 without the $3,000 capital-loss offset." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Reported $21,735, about one-fifth of the correct liability, with no bracket computation. On $503,494.75 of taxable income, ordinary bracket tax alone is $89,095.92, before the $11,688 qualified-dividend tax and $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Gave no numeric derivation and mixed standard and itemized deductions. Its $75,657 is below even the $89,095.92 ordinary-bracket tax on $425,574.75 of ordinary taxable income, so it understated the bracket tax and dropped part of the $11,688 qualified-dividend tax and the $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state tax refund as income and denied the $2,164 traditional IRA deduction, partly offset by excluding the full $46,308 of 401(k) deferrals instead of the $41,650 pro-rated to the $24,500 combined traditional+Roth limit. Taxable income came out at $505,148 versus $503,494.75, overstating tax by $529.02 at 32%." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Gave no numeric derivation. The correct path applies the $32,200 standard deduction plus $1,070 non-itemizer charity to AGI of $536,764.50, giving $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT. Its $104,535 is $1,971 short, which equals understating taxable income by about $6,160 at 32%; that fits excluding the full $46,308 of 401(k) deferrals instead of $41,650, plus other overstated deductions." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state refund as taxable and itemized, but PolicyEngine excludes the refund, and the $32,200 standard deduction plus $1,070 non-itemizer charity beats the phased-down ~$30.9k SALT itemization (the 0.5% AGI floor wipes out the itemized charity). Its $107,138 is $632 high, about $1,975 of excess taxable income at 32%." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $107,377 matches taxable income of about $506,218: AGI of $538,418 less only the bare $32,200 standard deduction. It counted the $4,148 refund, denied the $2,164 IRA deduction, excluded the full $46,308 of 401(k) deferrals instead of $41,650, and left out the $1,070 OBBBA non-itemizer cash charitable deduction, overstating taxable income by $2,723." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employee ESI premiums from wages (AGI $525,049) and kept the full $46,308 401(k) exclusion; PolicyEngine reduces wages only by the $41,650 of pro-rated 401(k) deferrals, giving AGI of $536,764.50. It then itemized $36,959.30 instead of the $33,270 standard-plus-non-itemizer deduction, understating taxable income by $15,405 and tax by $4,929.63." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 state refund as taxable, which PolicyEngine excludes, and took the bare standard deduction without the $1,070 OBBBA non-itemizer cash charitable deduction, overstating taxable income. Its $107,077 is $571 above the correct $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employee ESI premiums from wages to reach AGI of $525,049; PolicyEngine reduces wages only by the $41,650 of pro-rated 401(k) deferrals (AGI $536,764.50). It also itemized a phased-down SALT deduction instead of taking the $32,200 standard deduction plus $1,070 non-itemizer charity, understating tax by about $4,100." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employee ESI premiums from wages to reach AGI of $525,049 instead of $536,764.50 (only the $41,650 of pro-rated 401(k) deferrals reduce wages), and itemized $34,385.30. Taxable income came out $12,831 too low, cutting tax by $4,105.95 at 32%." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Cited a qualified business income deduction even though the household has no business income, and gave no numeric derivation. Its $112,450 overshoots by $5,944 the correct total: $89,095.92 ordinary tax + $11,688 15% qualified-dividend tax + $5,721.98 NIIT on $503,494.75 of standard-deduction taxable income." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: personal exemptions, Pease, 2%-floor miscellaneous deductions, and a pre-TCJA AMT exemption that produced $125,199 of tentative AMT. OBBBA permanently repealed those rules and set the 2026 AMT exemption at $140,200 with a $1,000,000 phase-out threshold, so no AMT applies and tax is $100,783.92 regular tax plus $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (Pease, miscellaneous itemized deductions, an AMT driven by SALT add-backs) and added $25,486 of AMT. Under OBBBA the TCJA rules are permanent, so the household takes the $32,200 standard deduction plus $1,070 non-itemizer charity and owes no AMT, since the $140,200 exemption phases out only above $1,000,000." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used the 2025 $31,500 standard deduction instead of 2026's $32,200, left out the $1,070 OBBBA non-itemizer cash charitable deduction, counted the $4,148 refund, denied the $2,164 IRA deduction, and excluded the full $46,308 of 401(k) deferrals instead of $41,650. That left taxable income at $506,918 versus $503,494.75." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Computed $106,958 of ordinary tax on $423,928, about $18,400 more than the 2026 OBBBA brackets give ($88,569), which points to a pre-TCJA rate schedule. It also subtracted $13,369 of employee ESI premiums from wages and itemized only $23,201 (including sunset-era 2%-floor miscellaneous deductions) instead of taking the larger $32,200 standard deduction plus $1,070 non-itemizer charity." -us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: uncapped ~$64k of SALT, 2%-floor miscellaneous deductions, Pease, and an $18,221 AMT add-on. Under OBBBA SALT is capped at a phased-down ~$30.9k, miscellaneous deductions stay suspended, Pease is repealed, and the $140,200 AMT exemption leaves no AMT. It also subtracted ESI premiums from wages (AGI $525,049 vs. $536,764.50)." -us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for this output, so nothing could be parsed. The correct computation is $89,095.92 ordinary tax + $11,688 qualified-dividend tax + $5,721.98 NIIT = $106,505.90." -us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $4,148 refund, denied the $2,164 IRA deduction, excluded the full $46,308 of 401(k) deferrals instead of the $41,650 pro-rated to the $24,500 combined traditional+Roth limit, and left out the $1,070 OBBBA non-itemizer cash charitable deduction. Taxable income came out at $506,218 versus $503,494.75, overstating tax by $871.42." -us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that itemized deductions, including mortgage interest on the $2.85M balance and unreimbursed employee expenses, wipe out taxable income. In fact no mortgage interest paid was listed, unreimbursed employee expenses are permanently non-deductible under OBBBA, and the SALT cap limits itemizing to ~$30.9k, so taxable income is $503,494.75 and tax is $106,505.90, not $0." -us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Reached taxable income of $506,218 instead of $503,494.75 by counting the $4,148 refund, denying the IRA deduction, using the unprorated $46,308 401(k) exclusion, and leaving out the $1,070 non-itemizer deduction. It then put NII at $142,667 instead of $150,578.34 (interest + dividends − $3,000 capital loss), giving NIIT of $5,421 rather than $5,721.98." -us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the entire $10,911 net capital loss in AGI instead of the $3,000 limit, invented $43,118 of mortgage interest from a balance with no stated interest, and netted the capital loss against qualified dividends (a $67,009 preferential base instead of $77,920). It also used wrong bracket breakpoints (24% ending at $247,775) and left out the $5,721.98 NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Applied a single-filer-sized $15,700 standard deduction instead of the $32,200 MFJ amount plus $1,070 non-itemizer charity, and understated gross income at $528,543 (correct AGI is $536,764.50). Its $89,067 covers roughly only regular tax and leaves out the $5,721.98 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Left out the 3.8% net investment income tax entirely ($5,722 on $150,578 of net investment income), which the frozen reference includes although the record leaves its inclusion in this output unresolved, and then added an unexplained ~$600 to its own $100,206. It also used a $32,600 standard deduction instead of $32,200 and skipped the $1,070 non-itemizer cash charitable deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its brackets, 15% dividend tax, and $5,722 NIIT match the frozen derivation, but taxable income is $559 above the frozen reference. Differences include the $1,070 non-itemizer charitable deduction, refund treatment, and the frozen engine's pro rata 401(k) allocation of $41,650 rather than $46,308; refund taxation and excess-deferral allocation are unresolved. Both the model and the frozen reference deduct $2,164 of IRA contributions, which the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $110,000 of mortgage interest from the $2.85M balance, although no interest paid was listed, so the mortgage-interest deduction is $0. Its $249,914 also has no arithmetic link to its own taxable-income figure. The frozen reference uses the $32,200 standard deduction plus the $1,070 non-itemizer deduction, giving $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax; those figures include a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Submitted 0 as a placeholder without computing anything, despite about $537k of AGI. The frozen computation gives $89,096 of ordinary tax on $425,575, $11,688 on $77,920 of qualified dividends, and $5,722 of NIIT; its taxable income includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly left out the $5,722 NIIT included in the frozen reference ('NIIT not part of this line'); the record leaves NIIT inclusion unresolved. It then reported $91,458, roughly its ordinary-bracket tax alone, dropping the $11,688 qualified-dividend tax it had computed itself (its own total was $102,907). It also used a $31,500 standard deduction and skipped the $1,070 non-itemizer charitable deduction. Its inclusion of the $4,148 refund differs from the frozen reference, but refund taxation is unresolved." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Imputed mortgage interest on the $2.85M balance although no interest paid was listed. It taxed qualified dividends at 20% even though income stacked on top of ordinary income stays in the 15% band. Its reported $154,430 also does not follow from its own components (~$92,000 + $15,584, about $107,600)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Mis-added AGI to $604,318, although its own components sum to about $538,418. It imputed mortgage interest that was never listed and applied an invented 18.8% blended rate to qualified dividends. The frozen reference's AGI is $536,764, followed by the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction; that AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its brackets, 15% dividend tax, and $5,722 NIIT match the frozen derivation, but taxable income ($504,054) is $559 above the frozen reference. Differences include the $1,070 non-itemizer charitable deduction, refund treatment, and the frozen engine's pro rata 401(k) allocation of $41,650 rather than $46,308; refund taxation and excess-deferral allocation are unresolved. Both the model and the frozen reference deduct $2,164 of IRA contributions, which the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expired in 2026, applying pre-TCJA brackets, a $10,000 SALT cap, restored 2% miscellaneous deductions, and a $7,141 AMT. Under 2026 law the TCJA bracket structure stays, the household takes the $32,200 standard deduction, and no AMT applies. It also imputed $52,500 of mortgage interest at an assumed 7% rate, with no interest listed, and computed NIIT on $153,578 without the $3,000 capital-loss offset (the correct base is $150,578)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Imputed about $28,000 of mortgage interest and used a $10,000 SALT cap in order to itemize, then reported a rounded $115,000 that was never worked through the brackets. With no mortgage interest listed, the household takes the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. The frozen reference reports $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax, including a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Itemized (SALT of about $31.9k plus $2.6k of charity) instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer cash charitable deduction. It ignored the 2026 floor of 0.5% of AGI, which wipes out the $2,574 itemized charitable deduction. Its refund inclusion and $46,308 of 401(k) deferrals differ from the frozen reference's refund exclusion and pro rata $41,650 allocation; the record leaves those treatments unresolved. Denying the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g), whereas the frozen reference improperly allows it." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used a $30,750 standard deduction instead of $32,200 and skipped the $1,070 non-itemizer charitable deduction. Its $507,668 taxable income exceeds the frozen reference's $503,495, which includes a defective $2,164 IRA deduction disallowed under 219(g); its refund inclusion also differs from the frozen reference, but refund taxation is unresolved. Its brackets also overstated ordinary tax ($92,234 on $429,748, versus $90,431 under the 2026 schedule). NIIT was computed on $153,578 without the $3,000 capital-loss offset." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Added a $131,247 AMT that does not arise: with the standard deduction and the 2026 AMT exemption, tentative minimum tax stays below regular tax. Its $465,449 taxable income reflects itemized deductions the facts do not support. NIIT used $153,578 instead of $150,578 of net investment income." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Applied rules that only hold if the TCJA expired: the Pease limitation, about $59.5k of itemized deductions, and an AMT that raised tax to $127,938. Under 2026 law the household takes the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction and owes no AMT. NIIT used $153,578 instead of $150,578." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Taxed $396,990 of ordinary income at pre-TCJA 2026 brackets after $63,508 of itemized deductions. 2026 law keeps the TCJA rate schedule (the 32% bracket starts at $403,550 for joint filers) and gives this household the $32,200 standard deduction plus the $1,070 non-itemizer deduction. Its NIIT on $153,578 also left out the $3,000 capital-loss offset." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expired: uncapped SALT, a $1M mortgage-interest limit despite no interest paid being listed, $10,400 of personal exemptions, and $108,283 of itemized deductions. Its $88,016 of ordinary tax therefore rests on the wrong deduction base. 2026 law gives the $32,200 standard deduction plus the $1,070 non-itemizer deduction. The frozen reference's $89,096 of ordinary tax on $425,575 also reflects a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Built AGI of $460,936 by counting pension income and dropping other taxable income, even though the $70,000 pension and the $8,961 of interest are tax-exempt and excluded. It gave no bracket work for its $110,996. The frozen reference's AGI is $536,764, leading to $89,096 + $11,688 + $5,722; it includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer-sponsored insurance premiums from wages, giving AGI of $525,049, compared with the frozen reference's $536,764. The frozen engine uses a pro rata $41,650 traditional 401(k) allocation and improperly deducts $2,164 of IRA contributions; the record leaves excess-deferral allocation unresolved and disallows the IRA deduction under 219(g). It then added an AMT that does not arise under 2026 law with the standard deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA individual provisions expired: it deducted uncapped state income and real estate taxes and used pre-TCJA brackets up to 33%. 2026 law keeps the TCJA brackets, and this household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction. The frozen reference's $89,096 ordinary tax, $11,688 dividend tax, and $5,722 NIIT also reflect a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. Its $106,093 is $413 below the frozen reference, consistent with the frozen tax structure applied to ordinary taxable income about $1,290 below the frozen amount. The frozen reference's $503,495 taxable income uses the engine's pro rata $41,650 traditional 401(k) allocation, $33,270 of standard plus charitable deductions, and a defective $2,164 IRA deduction disallowed under 219(g); excess-deferral allocation is unresolved." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted $13,369 of employer insurance premiums from wages (AGI $525,049, compared with the frozen reference's $536,764, which includes a defective $2,164 IRA deduction disallowed under 219(g)) and used itemized deductions and personal exemptions as if the TCJA had expired. It also taxed qualified dividends at 20%, although income stacked on top of ordinary income stays in the 15% band ($11,688 on $77,920)." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied brackets that assume the TCJA expired, subtracted health insurance premiums from wages, and itemized mortgage interest that was never listed, understating tax at $88,392. The frozen computation taxes $425,575 of ordinary income at 2026 TCJA-structure brackets ($89,096) and adds $11,688 of dividend tax and $5,722 of NIIT; its taxable income includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Itemized as if the TCJA had expired (uncapped state income tax, 2% miscellaneous deductions) instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Its $534,270 AGI uses $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation, which the prompt does not settle. It correctly omits the $2,164 IRA deduction disallowed under 219(g). Its rounded $105,000 differs from the frozen reference's $89,096 + $11,688 + $5,722, which includes that defective IRA deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Used 2025 figures (a $30,000 standard deduction and 2025 brackets) instead of the 2026 $32,200 standard deduction, the $1,070 non-itemizer charitable deduction, and 2026 brackets. It left out the $5,722 net investment income tax entirely and counted the $4,148 state refund in AGI." +us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed the full $2,000 non-itemizer charitable deduction although only $1,070 of cash donations exist. It deducted $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation, which the prompt does not settle. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g); the frozen reference improperly allows it. It also computed NIIT on $153,578 instead of $150,578, and its $87,724 of ordinary tax on $422,150 understates the 2026-bracket amount of $88,000." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Reported $21,735 with no bracket work, about one-fifth of the frozen reference. The frozen $503,495 taxable income produces $89,096 ordinary tax, $11,688 dividend tax, and $5,722 NIIT, but includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Reported $75,657 with no derivation, about $31,000 too low. The frozen computation taxes $425,575 of ordinary income at 2026 joint brackets ($89,096) and $77,920 of qualified dividends at 15% ($11,688), then adds $5,722 of NIIT; its taxable income includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used the $32,200 standard deduction plus the $1,070 non-itemizer deduction and the $5,722 NIIT, but its $538,418 AGI is $1,654 above the frozen reference's. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen refund exclusion and pro rata $41,650 allocation; those treatments are unresolved. Its answer is $163.42 below the adjudicated corrected $107,198.34." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Gave no numbers in its reasoning. Its $104,535 is $1,971 below the frozen reference, matching ordinary taxable income about $6,160 below the frozen $425,575. The frozen components are $89,096 ordinary tax, $11,688 dividend tax, and $5,722 NIIT, but include a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Included the $4,148 refund and itemized, differing from the frozen reference's refund exclusion and $32,200 standard deduction plus $1,070 charitable deduction; refund taxation is unresolved. Its tax is $632 above the frozen reference, whose $503,495 taxable income includes a defective $2,164 IRA deduction disallowed under 219(g). Its answer is $60.34 below the adjudicated corrected $107,198.34." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its total matches taxable income of $506,218 with the $32,200 standard deduction and no $1,070 non-itemizer charitable deduction. Its inferred $538,418 AGI includes the refund and uses $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its tax is $871 above the frozen reference, but only $178.66 above the adjudicated corrected $107,198.34." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted $13,369 of employer-sponsored insurance premiums from wages (AGI $525,049 compared with the frozen reference's $536,764) and itemized $36,959, producing taxable income below the frozen reference's. The frozen engine uses a pro rata $41,650 traditional 401(k) allocation and $32,200 standard plus $1,070 charitable deductions, but also improperly deducts $2,164 of IRA contributions disallowed under 219(g); excess-deferral allocation is unresolved." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Included the $4,148 refund and deducted $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its tax is $571 above the frozen reference, but $121.34 below the adjudicated corrected $107,198.34." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer insurance premiums from wages (AGI $525,049 compared with the frozen reference's $536,764) and itemized a phased-down SALT deduction instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Its tax is $4,106 below the frozen reference, whose AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer insurance premiums from wages (AGI $525,049 compared with the frozen reference's $536,764) and used $34,385 of itemized deductions instead of the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Its tax is $4,106 below the frozen reference, whose AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Cited a qualified business income deduction and mortgage interest that the facts do not support (there is no business income and no interest paid listed), and reported $112,450 without a derivation, about $5,944 above the frozen reference. The frozen components are $89,096 + $11,688 + $5,722 after the $32,200 standard deduction plus the $1,070 non-itemizer deduction, but also reflect a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied rules that only hold if the TCJA expired (Pease, personal exemptions, miscellaneous itemized deductions) and concluded that a $125,199 tentative AMT exceeds regular tax. Under 2026 law the household takes the standard deduction, the $140,200 joint AMT exemption is not phased down at this income, tentative minimum tax stays below regular tax, and no AMT applies." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied rules that only hold if the TCJA expired (Pease, $59,573 of itemized deductions) and added $25,486 of AMT from SALT and miscellaneous add-backs on top of regular tax. In 2026 the household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction and owes no AMT. The frozen total of $89,096 + $11,688 + $5,722 also reflects a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used the 2025 $31,500 standard deduction instead of 2026's $32,200 and skipped the $1,070 non-itemizer cash charitable deduction. Its $538,418 AGI includes the refund and deducts $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its $506,918 taxable income exceeds the defective frozen $503,495, and its answer is $332.66 above the adjudicated corrected $107,198.34." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted employer health premiums from wages and itemized as if the TCJA had expired (uncapped property tax, 2% miscellaneous deductions). Its $106,958 of ordinary tax on $423,928 far exceeds the 2026 schedule amount of about $88,569. It also computed NIIT on $153,578 instead of $150,578." +us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Applied rules that only hold if the TCJA expired (uncapped SALT of about $64,011, Pease, 2% miscellaneous deductions) and added $18,221 of AMT. Under 2026 law the household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction with no AMT. It also subtracted health premiums from wages, giving AGI of $525,049 compared with the frozen reference's $536,764, which includes a defective $2,164 IRA deduction disallowed under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for this output, so there is no answer to score." +us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Used the $32,200 standard deduction but missed the $1,070 non-itemizer cash charitable deduction. Its $538,418 AGI includes the refund and deducts $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its $506,218 taxable income exceeds the defective frozen $503,495; its tax is $871 above the frozen reference, but only $178.98 above the adjudicated corrected $107,198.34." +us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Concluded that deductions wipe out all tax by inventing mortgage interest on the $2.85M balance (no interest paid is listed) and counting unreimbursed employee business expenses, which are not deductible. The frozen reference reports $536,764 of AGI, $33,270 of standard plus non-itemizer deductions, and $106,506 of tax; those figures include a defective $2,164 IRA deduction that the adjudicated corrected $107,198.34 disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted employer insurance premiums from wages, putting AGI near $525,000 compared with the frozen reference's $536,764, and gave no derivation for the remaining steps. Its tax is $2,806 below the frozen reference, whose AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." +us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Missed the $1,070 non-itemizer cash charitable deduction. Its $538,418 AGI includes the refund and deducts $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g), whereas the frozen reference improperly allows it. It also computed NIIT on $142,667 instead of $150,578 ($5,421 instead of $5,722)." +us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Imputed $43,118 of mortgage interest at an assumed 5.75% rate although no interest paid was listed. It netted the $10,911 capital loss against qualified dividends, giving a preferential base of $67,009 instead of $77,920 and no $3,000 ordinary loss deduction. It also used invented bracket boundaries and left out the $5,722 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Used a $15,700 single-filer standard deduction on a joint return instead of $32,200 plus the $1,070 non-itemizer charitable deduction. It added an AMT that does not arise and omitted the $5,722 NIIT included in the frozen reference; NIIT inclusion in this output is unresolved. Its $89,067 is below the frozen $106,506, which includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g)." us,scenario_005,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_005,payroll_tax,claude-fable-5,llm_error,other,False,"The model's own working added up to $35,558.20, using a $183,600 wage base and CA SDI at 1.2%. It then submitted $33,296.55, a number its calculation doesn't support. Even its working total was low because the 2026 CA SDI rate is 1.3% ($5,590), not 1.2% ($5,160), and the Social Security wage base is $184,500." -us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It got all the FICA parts right ($22,599 + $6,235 + $1,620 = $30,454) but left out California's mandatory employee SDI. That tax is 1.3% of the full $430,000 with no cap, which adds $5,590." -us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 Social Security wage base of $168,600 for both earners, which also wrongly capped the head's $180,000. It applied SDI at 1% capped at $153,164, though SDI has been uncapped since 2024 and is 1.3% in 2026. It also added a made-up ETT amount and $10,086.52 of state income tax, none of which is employee payroll tax." -us,scenario_005,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"It never calculated anything and simply entered 0. The correct total is $22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare Tax + $5,590 CA SDI on $430,000 of wages." -us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It also capped the spouse's Social Security wages at $181,800 instead of $184,500, which understated that tax by $167." -us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It calculated FICA correctly at $30,454, then dropped CA SDI as uncertain. The correct SDI is 1.3% × $430,000 = $5,590. It then submitted $28,586.60, which is even below its own FICA total." -us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It mentioned CA SDI but wrongly said it was already included, then submitted roughly the FICA total alone ($30,437 vs. its own $30,454). That left out the $5,590 of SDI at 1.3% of all $430,000 in wages." -us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 Social Security wage base of $176,100 instead of the 2026 base of $184,500. That wrongly capped the head's $180,000 as well as the spouse's wages. It also used 1.2% for CA SDI instead of the 2026 rate of 1.3%." -us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its working reached $34,421.40 using a $176,100 wage base and SDI at 1.1%. It then submitted $27,306, which matches nothing it calculated. The correct figures are a $184,500 cap on the spouse only and SDI at 1.3% uncapped, which gives $5,590." -us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It charged Social Security once for the whole household ($11,253). Each spouse owes 6.2% on their own wages up to the cap: $11,160 for the head plus $11,439 for the spouse. It also left out CA SDI of $5,590." -us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its CA SDI of $4,080 is far below the correct $5,590, which is 1.3% of all $430,000 with no wage cap since 2024. Its Social Security figure of $22,407 also reflects a spouse wage base below the 2026 $184,500." -us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It capped CA SDI at $153,164 per worker at 1.2%, totaling $3,675.94. SB 951 removed the SDI wage ceiling starting in 2024, and the 2026 rate is 1.3%, so SDI is $5,590. It also used a $181,800 Social Security base instead of $184,500." -us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of 1.3% ($5,590). It also used a $184,200 Social Security wage base instead of $184,500." -us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590). Its Social Security figure of $21,242 implies it capped both earners near $171,000. The head's $180,000 falls below the 2026 base of $184,500 and is fully taxable." -us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only federal Social Security and Medicare and left out California's mandatory 1.3% employee SDI on $430,000, worth $5,590." -us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted the ESI premiums from FICA and SDI wages, but the engine taxes the full $180,000 and $250,000 of gross wages. Its SDI also falls short of 1.3% of the full $430,000." -us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied CA SDI at the outdated 1.1% ($4,730) instead of the 2026 rate of 1.3% ($5,590). Its total implies Social Security of about $21,115, which is below the correct $22,599 from the $184,500 cap applied to the spouse only." -us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out both the $1,620 Additional Medicare Tax and the $5,590 CA SDI. Its $26,588 is also below the correct FICA total of $30,454." -us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It cut the wage base to $173,111 and $243,520 by subtracting ESI premiums, but the engine applies payroll taxes to the full gross wages. It also used 1.1% for CA SDI instead of 1.3% of $430,000 ($5,590)." -us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $33,814 is $2,230 short. The correct figures are $22,599 + $6,235 + $1,620 + $5,590. That shortfall means its CA SDI was well below 1.3% of the full uncapped $430,000." -us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It left out California's mandatory 1.3% employee SDI ($5,590). It also capped both earners at the old $176,100 Social Security base, though the head's $180,000 falls below the 2026 base of $184,500." -us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It capped CA SDI at $165,384 per worker at 1.2% ($3,969.22). In 2026 SDI has no wage ceiling and the rate is 1.3%, so SDI is $5,590. It also used a $182,100 Social Security base instead of $184,500." -us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly said California has no mandatory employee state payroll tax. CA SDI is required at 1.3% of all wages in 2026, which adds $5,590 to its correct $30,454 of FICA." -us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It said no Additional Medicare Tax applies, but combined wages exceed the $250,000 MFJ threshold by $180,000, which adds $1,620. It also left out the $5,590 CA SDI and understated Social Security and Medicare below the correct $28,834." -us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $13,905 is less than half of the correct federal FICA ($30,454) alone. It treated the Social Security cap as limiting all FICA, even though Medicare's 1.45% applies to all $430,000. It also left out the $5,590 CA SDI." -us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It also used a $183,600 Social Security wage base instead of $184,500." -us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the right 1.3% SDI rate, but its $36,667 is $623 above the sum of $22,599 + $6,235 + $1,620 + $5,590. That means it overstated Social Security, either by applying a wage base above $184,500 to the spouse or by miscalculating." -us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $35,614 equals correct FICA ($30,454) plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3%, which gives $5,590." -us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $35,614 equals correct FICA ($30,454) plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3%, which gives $5,590." -us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $13,369 of ESI premiums as reducing wages, then applied Social Security, Medicare, Additional Medicare and 1.3% SDI to $416,631. The engine applies these taxes to the full $430,000 of gross wages." -us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It calculated CA SDI as $5,160 (1.2%) instead of 1.3% × $430,000 = $5,590. All other components ($22,599, $6,235, $1,620) were correct." -us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the ESI premiums as reducing wages, which shrank the base for FICA, Additional Medicare and 1.3% SDI to $416,631. The engine taxes the full $180,000 and $250,000 of gross wages." -us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its reasoning lists only OASDI, HI and Additional Medicare Tax and never mentions CA SDI. Its $34,515 is $1,529 short of the correct total, so it did not apply SDI as 1.3% of the full $430,000 ($5,590)." -us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It capped CA SDI and used 1.2%, getting $4,226. SDI has had no wage ceiling since 2024 and is 1.3% in 2026, which gives $5,590. It also used a spouse Social Security base of about $182,000 instead of $184,500." -us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% only up to an SDI wage limit. That limit was repealed in 2024, and the 2026 rate is 1.3% of all $430,000 ($5,590)." -us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted $13,369 of ESI premiums to get $416,631 of FICA wages, though the engine taxes the full $430,000. It also applied SDI at 1.1% to an invented $83,067 cap per person instead of 1.3% uncapped ($5,590)." -us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It reduced wages by the ESI premiums and capped Social Security at the 2024 base of $168,600 instead of $184,500. It also applied CA SDI to capped wages, though SDI is 1.3% of all $430,000." -us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score." -us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It got Social Security ($22,599), Medicare ($6,235) and Additional Medicare ($1,620) right, but used 1.2% for CA SDI ($5,160). The 2026 rate is 1.3%, which gives $5,590." -us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It claimed payroll taxes are offset by income-tax deductions and exemptions, but employee FICA and SDI are not reduced by income-tax deductions. Wages of $430,000 produce $22,599 Social Security, $6,235 Medicare, $1,620 Additional Medicare and $5,590 CA SDI." -us,scenario_005,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It said no CA employee payroll tax is modeled and so left out the 1.3% SDI ($5,590). It also used the $200,000 single-filer threshold on the spouse's wages for Additional Medicare Tax ($2,250). The correct rule applies the $250,000 MFJ threshold to combined wages, giving $1,620." -us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% up to a $153,164 cap per earner, even though the cap was repealed in 2024 and the 2026 rate is 1.3% of all $430,000. It also used the 2025 Social Security base of $176,100, which wrongly capped the head's $180,000." -us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It counted only federal FICA and left out the $5,590 CA SDI. Its $25,474, which it said included Additional Medicare Tax, is also below the correct FICA total of $30,454." +us,scenario_005,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It used 1.2% for CA SDI ($5,160) when the 2026 rate is 1.3% uncapped ($5,590), and a $183,600 Social Security base instead of $184,500. It then submitted $33,296.55, which matches neither its own $35,558.20 total nor any component sum." +us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Its federal figures are right ($22,599 + $6,235 + $1,620 = $30,454), but it left out California's mandatory employee SDI: 1.3% of $430,000, or $5,590." +us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It used the 2024 Social Security base of $168,600 and wrongly capped the head's $180,000 under it. It also applied a 1% SDI rate with a $153,164 ceiling that no longer exists, and then added the employer-paid ETT plus about $10,087 of state income tax, which are not employee payroll taxes." +us,scenario_005,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It skipped the calculation and entered $0, even though $430,000 of wages owes $30,454 in employee FICA and Additional Medicare Tax plus $5,590 in CA SDI." +us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on all wages ($5,590). It also capped the spouse's Social Security at $181,800 instead of the $184,500 2026 base, understating that tax by $167." +us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly found $30,454 of federal employee tax, then dropped CA SDI (1.3% × $430,000 = $5,590). It submitted $28,586.60, a number below even its own FICA subtotal." +us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It said CA SDI was 'already reflected' but submitted $30,437, which is just the federal FICA total (and $17 short of $30,454). The $5,590 of 1.3% uncapped SDI was never added." +us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 Social Security base of $176,100, which wrongly capped the head's $180,000 and undercounted the spouse (2026 base is $184,500). It also used 1.2% for CA SDI instead of 1.3%, understating SDI by $430." +us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It used the old $176,100 Social Security base for both earners and a 1.1% SDI rate instead of 1.3% on $430,000. It then submitted $27,306, which does not match its own $34,421.40 working total." +us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It counted Social Security for only one worker ($11,253) instead of $11,160 for the head plus $11,439 for the spouse. It also left out CA SDI of $5,590 (1.3% of $430,000)." +us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its CA SDI of $4,080 is too low; the 2026 rule is 1.3% of all $430,000 of wages, or $5,590. Its Social Security of $22,407 also implies a spouse wage base below the $184,500 2026 cap." +us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It capped CA SDI wages at $153,164 per worker, a ceiling repealed for 2024 onward, and used 1.2% instead of 1.3%, giving $3,676 instead of $5,590. It also capped the spouse's Social Security at $181,800 instead of $184,500." +us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used 1.2% for CA SDI ($5,160) instead of the 2026 rate of 1.3% ($5,590), and a $184,200 Social Security base instead of $184,500." +us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied CA SDI at the old 1.1% rate ($4,730) instead of 1.3% ($5,590). Its Social Security of $21,242 is $1,357 below the correct $22,599 ($11,160 + $11,439 at the $184,500 base)." +us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out California's mandatory employee SDI ($5,590 at 1.3%). Its $31,398 also does not match the correct federal sum of $30,454." +us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It took the employer health premiums out of the wage base. PolicyEngine taxes the full $430,000 gross for Social Security, Medicare, Additional Medicare and SDI, and SDI is 1.3% of that full amount, or $5,590." +us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% ($4,730) instead of the 2026 rate of 1.3% on all wages ($5,590). Its total also implies Social Security below the correct $22,599." +us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out both the $1,620 Additional Medicare Tax on joint wages above $250,000 and the $5,590 CA SDI. Even its FICA figure is below the correct $28,834 of base Social Security and Medicare." +us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It cut each worker's wages by the employer health premiums, to $173,111 and $243,520, when the full $430,000 gross is taxable. It also used a $181,200 Social Security base instead of $184,500 and a 1.1% SDI rate instead of 1.3%." +us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"The correct federal total is $30,454, so its $33,814 implies only about $3,360 of CA SDI. The correct SDI is 1.3% of all $430,000 of wages, or $5,590, which means it used too low a rate or a repealed wage cap." +us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It left out CA SDI ($5,590 at 1.3% uncapped). It also used the 2025 base of $176,100 for both workers, which wrongly capped the head's $180,000 and undercounted the spouse against the $184,500 2026 base." +us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It capped CA SDI wages at $165,384 per worker at 1.2%, but California removed the SDI wage ceiling in 2024 and the 2026 rate is 1.3%, so SDI is $5,590, not $3,969. It also capped the spouse's Social Security at $182,100 instead of $184,500." +us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It stated that California has no mandatory employee payroll tax. California requires employee SDI at 1.3% of all wages in 2026, which adds $5,590 to its correct federal $30,454." +us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It wrongly said the Additional Medicare Tax does not apply. Joint Medicare wages of $430,000 exceed the $250,000 married-filing-jointly threshold by $180,000, so $1,620 is owed. It also left out CA SDI ($5,590), and its Social Security plus Medicare is below the correct $28,834." +us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $13,905 roughly covers one worker's FICA. The correct total is $22,599 Social Security for both earners, $6,235 Medicare on $430,000, $1,620 Additional Medicare and $5,590 CA SDI." +us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It used 1.2% for CA SDI ($5,160) instead of the 2026 rate of 1.3% ($5,590), and a $183,600 Social Security base instead of $184,500, which understated the spouse's tax by $55.80." +us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the correct 1.3% SDI rate ($5,590), but its $36,667 implies $31,077 of federal tax, $623 more than the correct $30,454. That overstatement matches a Social Security wage base about $10,000 above the $184,500 2026 cap." +us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $35,614 equals the correct $30,454 federal total plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3% of all wages, or $5,590." +us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $35,614 equals the correct $30,454 federal total plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3% of all wages, or $5,590." +us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the employer health premiums as pretax and cut payroll wages to $416,631, which lowered Social Security, Medicare, Additional Medicare and SDI. The full $430,000 gross is taxable, so Medicare is $6,235, Additional Medicare is $1,620 and SDI is $5,590." +us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It computed CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on all $430,000 of wages ($5,590). Its federal figures ($22,599, $6,235, $1,620) are correct." +us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the employer health premiums as pretax and cut payroll wages to $416,631, which shrank every component. The full $430,000 gross is taxable, giving $22,599 Social Security, $6,235 Medicare, $1,620 Additional Medicare and $5,590 SDI." +us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its reasoning lists only Social Security, Medicare and Additional Medicare Tax, with no explicit CA SDI. Its $34,515 is $4,061 above the correct $30,454 federal total, which falls short of the $5,590 SDI owed at 1.3% uncapped." +us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied a wage cap to CA SDI at 1.2% ($4,226), but the cap was repealed in 2024 and the 2026 rate is 1.3% of all wages ($5,590). It also capped the spouse's Social Security at about $182,000 instead of $184,500." +us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% subject to a wage limit. In 2026 SDI has no wage limit and the rate is 1.3%, giving $5,590 on $430,000." +us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It cut payroll wages by the $13,369 of health premiums when the full $430,000 gross is taxable. It also applied CA SDI at 1.1% to an invented $83,067 per-person base instead of 1.3% of all wages ($5,590)." +us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It cut wages by the health premiums and capped Social Security at the 2024 base of $168,600 instead of $184,500. It also capped CA SDI at 1.2%, when the 2026 SDI is 1.3% of the full $430,000 with no cap." +us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there is no answer to score." +us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Its Social Security, Medicare and Additional Medicare figures are correct ($30,454), but it used 1.2% for CA SDI ($5,160) instead of the 2026 rate of 1.3% ($5,590)." +us,scenario_005,payroll_tax,minimax-m3,llm_error,other,False,"It claimed employee payroll taxes are 'offset by deductions/exemptions' and entered $0. Income tax deductions do not reduce FICA or SDI, and $430,000 of wages owes $36,044 in employee payroll tax." +us,scenario_005,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"It took cafeteria-plan health premiums out of the payroll wage base. The full $430,000 gross is taxable for Medicare, Additional Medicare and 1.3% CA SDI ($5,590)." +us,scenario_005,payroll_tax,ox-alpha,llm_error,household_unit_or_filing_status,False,"It used the $200,000 per-employee withholding threshold on the spouse's wages ($2,250) instead of 0.9% of the couple's combined wages above the $250,000 joint threshold ($1,620). It also said California has no mandatory employee payroll tax and left out the $5,590 SDI." +us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% with a repealed $153,164 per-worker cap ($3,370) instead of 1.3% of all wages ($5,590). It also capped both workers at the 2025 $176,100 Social Security base, including the head's $180,000." +us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It left out CA SDI entirely, and its $25,474 of federal tax is short of the correct $30,454 ($22,599 Social Security on $180,000 plus $184,500, $6,235 Medicare, $1,620 Additional Medicare)." us,scenario_005,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It invented about $22,800 of mortgage-interest deduction from the loan balance, even though no interest paid was listed, so that amount should be $0. It also deducted the full $46,308 of 401(k) deferrals despite the $24,500 combined traditional+Roth limit. Its stated tax of about $38,832 was then replaced with an unexplained $45,856 that does not follow from its own working." -us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its California AGI of $532,106 deducts the full $46,308 of traditional 401(k) deferrals and also the $2,164 of IRA contributions. The 401(k) exclusion is capped at about $20,825 per spouse under the $24,500 combined limit, and the IRA contributions are not deductible for covered workers at this income. The correct California AGI is $538,928, and the surviving exemption credit is $114, not about $210." -us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted a made-up deduction for federal income tax paid, which California does not allow, and also subtracted employer-paid insurance premiums. It applied a 12% bracket that does not apply at $518k, and it invented about $21,621 of nonrefundable credits when only a $114 exemption credit applies." -us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed the California tax and entered $0 as a placeholder. California taxable income is $518,408, which produces $40,975 of tax after the $114 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed about $60,000 of mortgage interest from the $2.85M balance even though no interest paid was given, so that amount should be $0. It left out the miscellaneous employee-expense deduction and deducted the full 401(k) deferrals despite the $24,500 combined limit. It also claimed a $560 exemption credit that phases down to $114 at this AGI." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension to California income, but it is excluded. It also imputed $35,000 of mortgage interest when none was listed. The final $36,767 does not follow from its own taxable income of about $595,000." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put part of the tax-exempt pension into California income and arrived at about $560,000 of taxable income instead of $518,408. It then applied rates up to 11.3%, but the top slice of this income is taxed at 9.3%." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its California AGI of $532,106 subtracts the full $46,308 of 401(k) deferrals and the non-deductible $2,164 of IRA contributions. The 401(k) exclusion is capped at about $20,825 per spouse because traditional plus Roth desires exceed the $24,500 limit. As a result, taxable income came out about $8,756 too low, and it used a $256 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It excluded the full $23,154 traditional 401(k) per spouse and ignored the $24,500 combined traditional+Roth cap, which understates California AGI by $4,658. It never applied the 6% high-income itemized-deduction limitation, which cuts $2,320. It also subtracted an unphased $500 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It assumed $50,000 of deductible mortgage interest even though no interest paid was given, so that amount should be $0. It left out the miscellaneous employee-expense deduction. Its bracket arithmetic also gave about $32,000, which is too low even for its own $464,416 of taxable income." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It started from California AGI of about $534k and deducted the full 401(k) deferrals despite the $24,500 combined limit. It left out the $2,986 deduction for employee business expenses above the 2% floor and skipped the 6% itemized limitation. It also treated the exemption credits as fully phased out, when $114 survives." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its taxable income of $518,564 is nearly correct, but its bracket computation gave $42,151 instead of the roughly $41,100 that the 2026 MFJ schedule produces. It also said no nonrefundable credits apply and dropped the $114 exemption credit that survives the phase-out." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its California taxable income of $497,700 is $20,708 below the correct $518,408, which is California AGI of $538,928 minus $20,520 of limited itemized deductions. It also subtracted a $300 exemption credit that phases down to $114 at this AGI." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its taxable income of $512,889 is $5,519 too low. That gap comes mostly from excluding the full $46,308 of traditional 401(k) deferrals instead of the roughly $41,650 allowed under the $24,500 combined traditional+Roth limit. It also wrongly said no nonrefundable credits apply, when a $114 exemption credit does." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used California AGI of $534,270, which deducts the full 401(k) deferrals despite the $24,500 combined limit and is $4,658 too low. It left out the $2,986 deduction for employee business expenses above the 2% floor, giving $18,663 of itemized deductions instead of $20,520. It also took a $310 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $73,017 of itemized deductions includes about $50,000 of mortgage interest that it imputed from the loan balance, even though no interest paid was listed. Without it, the limited itemized total is $20,520. It also used the $534,270 AGI, which ignores the 401(k) combined-limit cap." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $31,215 is about $9,760 short, which is consistent with understating taxable income by roughly $105,000. The correct calculation is California AGI of $538,928 minus $20,520 of limited itemized deductions, giving $518,408 of taxable income, then the 2026 MFJ schedule and the $114 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $501,848 of California taxable income is $16,560 below the correct $518,408. That figure matches subtracting the $13,369 of employer-paid insurance premiums (not part of reported wages) and taking unlimited itemized deductions. Its bracket math then overstated tax on that smaller base, so the two errors only partly cancel." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its California AGI of $520,901 equals $534,270 minus the $13,369 of employer-sponsored insurance premiums. Those premiums are employer-paid and were never in gross wages. It also deducted the full 401(k) deferrals despite the $24,500 combined limit, and it left the $2,986 employee-expense deduction out of its $19,854 itemized total." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $33,816 is about $7,160 short, which implies roughly $77,000 too little taxable income. The correct taxable income is $518,408 (California AGI of $538,928 minus $20,520 of limited itemized deductions), with a $114 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It started from federal AGI without the $4,148 California subtraction for the state tax refund. It did not apply the 6% high-income itemized limitation or the $114 exemption credit, which overstated tax by about $1,175." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $36,918 is about $4,057 short, which implies roughly $43,600 too little taxable income, consistent with imputing deductions that were not listed, such as mortgage interest. The correct base is California AGI of $538,928 minus $20,520 of limited itemized deductions." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used California AGI of $534,270, which deducts the full $46,308 of 401(k) deferrals. Because traditional plus Roth desires exceed the $24,500 per-person limit, only about $41,650 is excludable, and the correct California AGI is $538,928. It then rounded the result to a flat $40,000." -us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $77,920 of qualified dividends out of AGI and deducted the full $10,911 net capital loss instead of the $3,000 limit. It then restricted the California capital loss to $2,000, but the MFJ limit is $3,000. It also used an $11,000 standard deduction even though itemized deductions of $20,520 are larger." -us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the roughly $11,300 standard deduction instead of itemizing. Property tax, charity and employee expenses above the 2% floor total $20,520 even after the high-income limitation. It also deducted the full 401(k) deferrals despite the $24,500 combined limit and dropped the $114 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no computation, and its $20,677 is about half the correct amount. California taxable income of $518,408, with the top slice taxed at 9.3%, produces $41,089 of tax before the $114 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $37,099 is $3,876 short, which implies roughly $41,700 too little taxable income compared with the correct $518,408. That figure is California AGI of $538,928 minus $20,520 of limited itemized deductions." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It took pretax health premiums out of wages, but the listed employer-sponsored premiums are employer-paid and were never in the reported gross wages. It also excluded the full 401(k) deferrals despite the $24,500 combined traditional+Roth limit. Both errors understate California AGI." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its California taxable income of $519,427 is $1,019 above the correct $518,408 (California AGI of $538,928 minus $20,520 of itemized deductions after the 6% limitation). That gap accounts for its $49 overstatement." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $41,150 is higher than the $41,089 tax before any credit, so it implies about $1,900 more taxable income than the correct $518,408. That means it undercounted the $20,520 of limited itemized deductions, which include $2,986 of employee expenses above the 2% floor." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated the $8,961 of federally tax-exempt interest as taxable in California, which PolicyEngine does not do. Even so, it came out $809 low, which implies taxable income about $8,700 below $518,408. That means it over-subtracted elsewhere, including excluding the full 401(k) deferrals despite the $24,500 combined limit." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its California AGI of $520,901 subtracts the $13,369 of employer-sponsored premiums, which are employer-paid and not in gross wages. It also subtracts the full $46,308 of 401(k) deferrals instead of the roughly $41,650 allowed under the $24,500 combined limit. It used a $222 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It kept the $4,148 state-tax refund in California income instead of subtracting it. It also used the standard deduction even though California itemized deductions of $20,520 after limitation are larger than the $11,412 standard deduction." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its California taxable income of $492,027 is $26,381 below the correct $518,408, which means its itemized deductions far exceed the allowable $20,520. It also applied two full personal exemption credits instead of the $114 left after the phase-out." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its California AGI of $520,901 subtracts the $13,369 of employer-paid insurance premiums, which were never in gross wages. It also subtracts the full 401(k) deferrals despite the $24,500 combined traditional+Roth limit. It used a $222 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $48,720 is $7,745 too high, which implies about $83,000 of extra taxable income. That is consistent with adding the $70,000 tax-exempt pension and the $8,961 of tax-exempt interest to California income, and neither belongs there." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used California AGI of $534,270, which excludes the full $46,308 of 401(k) deferrals and ignores the $24,500 combined traditional+Roth cap that limits the exclusion to about $20,825 per spouse. It also took about $280 of exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It took $22,934 of itemized deductions without the 6% high-income limitation, which reduces them by $2,320 to $20,520. It also skipped the $4,148 SALT-refund subtraction, and it used a $177 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $2,986 deduction for employee business expenses above the 2% floor. It used California AGI of $534,270, which excludes the full 401(k) deferrals despite the $24,500 combined limit and is $4,658 too low. It also took a $314 exemption credit instead of the $114 left after the phase-out." -us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its California AGI of $525,049 is federal AGI of $538,418 minus the $13,369 of employer-paid premiums, which are not in wages. It also skipped the SALT-refund subtraction and did not apply the 6% itemized limitation. It took no exemption credit, when $114 survives the phase-out." -us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension and the $8,961 of tax-exempt interest to California AGI and got about $604,010. Both are excluded, and the correct California AGI is $538,928." -us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer and no explanation for state_income_tax_before_refundable_credits, so there was no value to score." -us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $538,418 excludes the full $46,308 of traditional 401(k) deferrals. Each spouse's $27,240 traditional+Roth desire exceeds the $24,500 limit, so only about $20,825 per spouse is excludable. That left California AGI $4,658 too low, and it used a $258 exemption credit instead of $114." -us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the federal $10,000 SALT cap to California, which does not have that cap, and it invented mortgage interest that was never listed. It then concluded that taxable income was eliminated, when California taxable income is $518,408." -us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It over-applied the high-income itemized limitation and cut $19,854 down to $11,912, but the 6%-of-excess-AGI reduction is only $2,320. It also left out the $2,986 employee-expense deduction and skipped the $4,148 SALT-refund subtraction." -us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It deducted the full $10,911 net capital loss instead of $3,000, did not subtract any 401(k) deferrals, and used the standard deduction instead of $20,520 of itemized deductions. Its bracket math then gave only $33,544 on $565,375 of taxable income, which is far below what the 9.3% bracket produces. It also subtracted an unphased $892 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its bracket computation gave $29,423 on $495,603 of taxable income, but the 2026 MFJ schedule with 9.3% on the top slice produces about $39,000 there. It also used the standard deduction instead of the larger $20,520 of itemized deductions." +us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It imputed about $22,800 of mortgage interest from the $2.85M balance, but no interest paid is listed, so the deduction is $0. It also subtracted the full $46,308 of traditional 401(k) deferrals instead of the $41,650 left after the $24,500 combined traditional+Roth cap. Its own bracket math reached about $38,832, then it jumped to $45,856 with no computation behind it and assumed the exemption credits were fully phased out, when $114 remains." +us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It set CA AGI at about $532,106 by subtracting the full $46,308 of traditional 401(k) deferrals and also deducting $2,164 of IRA contributions. The engine allows no IRA deduction and caps each spouse's traditional deferral at $20,825, so CA AGI is $538,928. That left taxable income about $7,900 short ($510,546 vs $518,408), and an exemption credit of about $210 instead of $114 pulled the tax further down." +us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It made up deductions for employer health premiums and 'federal income tax paid', used a 12% bracket that does not apply at this income, and subtracted about $21,621 of nonrefundable credits that do not exist. The only nonrefundable credit here is the $114 phased-down exemption credit, taken against $41,089 of tax on $518,408 of taxable income." +us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It did no calculation and entered $0. The household owes $41,089 of California tax on $518,408 of taxable income, less a $114 exemption credit." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed about $60,000 of mortgage interest from the $2.85M balance, but no interest paid is listed, so the deduction is $0. It also left out the miscellaneous employee-expense deduction and the high-income itemized limitation, and subtracted the full $46,308 of 401(k) deferrals, giving $453,116 of taxable income instead of $518,408. It took a $560 exemption credit with no phase-down, when only $114 is allowed at this AGI." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension to California income and imputed about $35,000 of mortgage interest with no interest input; the engine excludes the pension and allows no mortgage interest. Its stated ~$595,000 of taxable income would produce far more than $36,767 at the 9.3% rate, so its answer follows neither its own figures nor the engine's $518,408 of taxable income." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put CA taxable income at about $560,000 by partly including the tax-exempt private pension and counting mortgage interest, and it cited rates up to 11.3%. The engine excludes the pension, allows no mortgage interest because none is listed, and taxes $518,408 with a 9.3% top marginal rate." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It computed CA AGI of about $532,106 by subtracting the full $46,308 of traditional 401(k) deferrals plus $2,164 of IRA contributions. The engine caps traditional deferrals at $20,825 per spouse because traditional plus Roth requests ($27,240) exceed $24,500, and it allows no IRA deduction, so CA AGI is $538,928. Taxable income came out about $8,750 low, and a $256 exemption credit instead of $114 left the answer $1,164 short." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the full $46,308 of traditional 401(k) deferrals instead of the $41,650 allowed under the $24,500 combined traditional+Roth cap, giving CA AGI of $534,270 instead of $538,928. It never applied California's high-income itemized limitation, so it kept $23,017 of deductions where the engine allows $20,520. It also took a ~$500 exemption credit with no AGI phase-down, when only $114 is allowed." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It imputed $50,000 of mortgage interest from the $2.85M balance even though no interest paid is listed, which cut taxable income to $464,416. It then misapplied its own brackets: that income yields roughly $37,000, not $32,000. The engine's itemized deductions are property tax, charity, and miscellaneous expenses ($20,520 after limitation), giving $518,408 of taxable income." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used CA AGI of about $534k by subtracting the full $46,308 of traditional 401(k) deferrals instead of the $41,650 allowed under the cap. It left out the $2,986 deduction for employee expenses above the 2% floor and treated the exemption credits as fully phased out when $114 remains. Taxable income ended about $4,400 below $518,408 and the result $475 short." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It started from $538,418 of federal AGI without the SALT-refund subtraction and itemized only $19,854, which happened to put taxable income ($518,564) almost exactly at the engine's $518,408. Its bracket math then produced $42,151 instead of about $41,100, and it dropped the $114 exemption credit that survives the phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $497,700 of taxable income is about $20,700 below the engine's $518,408. That gap comes from deducting more than the engine's $41,650 of capped 401(k) deferrals and $20,520 of limited itemized deductions from $538,928 of CA AGI. It also took a $300 exemption credit instead of the $114 left after phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $512,889 of taxable income is $5,519 below the engine's $518,408. Most of the gap comes from subtracting the full $46,308 of traditional 401(k) deferrals instead of the $41,650 allowed once traditional plus Roth requests are capped at $24,500 per spouse. It also claimed no exemption credit, though $114 survives the phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270 by subtracting the full $46,308 of 401(k) deferrals instead of the capped $41,650. It itemized only $18,663 because it left out the $2,986 of employee expenses above the 2% floor, giving $515,607 of taxable income instead of $518,408. It also took a $310 exemption credit with no phase-down, when $114 is allowed." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $73,017 of itemized deductions includes imputed mortgage interest, but no interest paid is listed, so the engine allows $0. It also skipped the high-income itemized limitation and subtracted the full $46,308 of 401(k) deferrals, cutting taxable income to $461,253 instead of $518,408." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. $31,215 corresponds to roughly $412,000 of taxable income, about $106,000 below the engine's $518,408 ($538,928 CA AGI less $20,520 of limited itemized deductions), which matches subtracting a large deduction that does not exist, such as mortgage interest, or leaving out investment income." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its ~$501,848 of taxable income is $16,560 below the engine's $518,408. It understated CA AGI relative to $538,928 (the engine excludes only $41,650 of capped 401(k) deferrals) and deducted more than the $20,520 left after the high-income itemized limitation." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It computed CA AGI of $520,901 by subtracting the $13,369 of employer-sponsored insurance premiums from wages on top of the full $46,308 of 401(k) deferrals. Those premiums are not a wage reduction here, and traditional deferrals are capped at $20,825 per spouse, so CA AGI is $538,928. It also itemized only $19,854, leaving out the $2,986 deduction for employee expenses above the 2% floor." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. $33,816 is $7,159 below the reference, which at the 9.3% marginal rate means taxable income about $77,000 below the engine's $518,408. That matches over-deducting from the $538,928 CA AGI, which carries only $20,520 of limited itemized deductions and no mortgage interest." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It started from federal AGI without subtracting the $4,148 SALT refund that California excludes, skipped the high-income itemized limitation, and applied no exemption credit, coming out $1,175 high. The engine taxes $518,408 for $41,089 and then subtracts the $114 phased-down exemption credit." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no figures, and its $36,918 is $4,057 low, about $43,600 less taxable income than the engine's $518,408 at 9.3%. That matches subtracting 'pre-tax' items beyond the engine's $41,650 of capped 401(k) deferrals, such as employer health premiums and IRA contributions, which the engine does not deduct." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270 by subtracting the full $46,308 of traditional 401(k) deferrals instead of the $41,650 allowed under the $24,500 combined traditional+Roth cap, giving about $515,000 of taxable income instead of $518,408. It then rounded the tax to $40,000 instead of computing $41,089 less the $114 exemption credit." +us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $77,920 of qualified dividends out of AGI, applied the full $10,911 net capital loss instead of the $3,000 limit, and invented a $2,000 California capital-loss limit. It also took an $11,000 standard deduction even though itemized deductions ($20,520 after limitation) are larger. Its $438,439 of taxable income is about $80,000 below the engine's $518,408." +us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the ~$11,300 standard deduction instead of itemizing $17,280 of property tax, $2,574 of charity, and $2,986 of miscellaneous expenses, which total $20,520 after the high-income limitation. It also subtracted the full $46,308 of 401(k) deferrals, and it applied no exemption credit even though $114 survives the phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It did no computation, and $20,677 is about half the correct figure. The engine taxes $518,408 of taxable income ($538,928 CA AGI less $20,520 of limited itemized deductions) for $41,089 and subtracts only a $114 exemption credit." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $37,099 is $3,876 low, about $41,700 of understated taxable income at 9.3%, which means it deducted far more than the engine's $41,650 of capped 401(k) deferrals and $20,520 of limited itemized deductions allow." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It reduced wages by 'pretax health premiums' ($13,369 of employer-sponsored premiums, which are not a wage reduction here) in addition to the full $46,308 of 401(k) deferrals. That understated CA AGI against the engine's $538,928, where traditional deferrals are capped at $20,825 per spouse, and left the answer $2,141 low." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its ~$519,427 of taxable income is about $1,019 above the engine's $518,408, which is $538,928 of CA AGI less $20,520 of limited itemized deductions. The extra income added about $95 of 9.3% tax, which a slightly oversized exemption credit only partly offset, leaving it $49 high." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied unspecified 'California additions', but the engine's only California adjustment to federal AGI is the $4,148 SALT-refund subtraction. Its result is $175 high, the equivalent of about $1,900 of extra taxable income at 9.3% above the engine's $518,408." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the $8,961 of tax-exempt interest to California income, which the engine does not do. It still finished $809 low, so it over-deducted elsewhere by more than that addition, which matches subtracting the full $46,308 of traditional 401(k) deferrals instead of the capped $41,650 and taking more than the $114 exemption credit left after phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It set CA AGI at $520,901 by subtracting $13,369 of employer-sponsored insurance premiums from wages and excluding the full $46,308 of 401(k) deferrals; the engine's CA AGI is $538,928, with traditional deferrals capped at $20,825 per spouse. It used the 2025 bracket schedule and exemption amounts that the release's California convention (c_ca_hold_2025) holds for 2026; its $222 exemption credit reflects its understated AGI, compared with $114 in the current reference." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It counted the $4,148 state tax refund as California income, though California subtracts it. It also took the standard deduction instead of itemizing $20,520 of property tax, charity, and miscellaneous expenses, which overstated taxable income and pushed the tax $815 too high." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its ~$492,027 of taxable income is $26,381 below the engine's $518,408, the result of deducting well beyond the $20,520 of limited itemized deductions (property tax, charity, miscellaneous) taken from $538,928 of CA AGI. It also took two full personal exemption credits instead of the $114 left after the high-AGI phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It set CA AGI at $520,901 by subtracting $13,369 of employer-sponsored insurance premiums from wages and excluding the full $46,308 of 401(k) deferrals; the engine's CA AGI is $538,928, with traditional deferrals capped at $20,825 per spouse. It used the 2025 bracket schedule and exemption amounts that the release's California convention (c_ca_hold_2025) holds for 2026; its $222 exemption credit reflects its understated AGI, compared with $114 in the current reference." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no breakdown. $48,720 is $7,745 high, which means taxable income about $83,000 above $518,408. That is the size of adding the $70,000 tax-exempt private pension and $8,961 of tax-exempt interest, both of which the engine keeps out of CA AGI." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270 by subtracting the full $46,308 of traditional 401(k) deferrals instead of the $41,650 left after the $24,500 combined traditional+Roth cap, giving taxable income of about $512,000 instead of $518,408. It also kept a ~$280 exemption credit where the phase-down leaves $114." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It treated CA AGI as equal to its $538,418 federal AGI without subtracting the SALT refund, and that federal AGI already subtracted the full $46,308 of 401(k) deferrals. Its $22,934 of itemized deductions skipped the $2,320 high-income limitation. That gave $515,484 of taxable income instead of $518,408, and with a $177 credit instead of $114 it finished $500 low." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270 by subtracting the full $46,308 of 401(k) deferrals instead of the capped $41,650, and it left out the $2,986 deduction for employee expenses above the 2% floor. It also took $314 of exemption credits where the phase-down leaves $114, which put the answer $318 low." +us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It set CA AGI at $525,049, about $13,900 below the engine's $538,928, and applied no high-income limitation to its $23,201 of itemized deductions, giving $501,848 of taxable income instead of $518,408. It also used no exemption credit, though $114 survives the phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension and the tax-exempt interest to CA AGI, reaching $604,010. The engine excludes both; its CA AGI is federal AGI less the SALT refund, $538,928, which leaves $518,408 of taxable income instead of $582,388." +us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the $40,975.25 reference." +us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its $538,418 federal AGI subtracts the full $46,308 of traditional 401(k) deferrals, so its CA AGI was $534,270 instead of $538,928; the engine allows only $20,825 per spouse because traditional plus Roth requests exceed $24,500. Taxable income came out at $512,433 instead of $518,408, and a $258 exemption credit instead of $114 left the answer $874 low." +us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the federal $10,000 SALT cap to California, which has no such cap, and imputed mortgage interest with no interest input, then concluded the tax was eliminated. California taxable income is $518,408, producing $41,089 of tax before the $114 exemption credit." +us,scenario_005,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It gave no figures. $39,300 is $1,675 low, which at 9.3% means taxable income about $18,000 below the engine's $518,408. That matches subtracting the full $46,308 of 401(k) deferrals instead of the capped $41,650 and deducting beyond the $20,520 of limited itemized deductions." +us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used $538,418 of federal AGI without subtracting the $4,148 SALT refund. It also over-applied the high-income itemized limitation, cutting $19,854 to $11,912, when the limitation is 6% of AGI above roughly $500k, or $2,320. It also left out the $2,986 miscellaneous deduction, so taxable income reached $526,500 instead of $518,408." +us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It never subtracted 401(k) deferrals, kept the $4,148 SALT refund in income, used the full $10,911 net capital loss instead of the $3,000 limit, and took the standard deduction instead of $20,520 of itemized deductions. It then miscomputed the brackets, getting $33,544 on $565,375 when the 9.3% schedule gives roughly $45,000, and subtracted $892 of exemption credits with no phase-down." +us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It took the standard deduction instead of $20,520 of itemized deductions (property tax, charity, and miscellaneous expenses after the high-income limitation). Its bracket math also produced only $29,423 on $495,603 of taxable income, when California's schedule with a 9.3% top marginal rate gives roughly $39,000." us,scenario_005,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Treated SSDI as entirely non-taxable. Because provisional income of $44,922.50 is above the $34,000 single threshold, $13,784.12 of the SSDI is taxable. It also used the 2024 $14,600 standard deduction instead of the 2026 $16,100 deduction." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Assumed taxable SSDI was about $0 because other income was 'low.' The $35,280 pension raises provisional income to $44,922.50, which makes $13,784.12 taxable and AGI $49,064.12. It therefore taxed only about $19,180 instead of $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Correctly derived taxable income of $32,964 but used a $12,150 top for the 10% bracket instead of $12,400, which gave $3,713. It then switched to a $16,600 deduction and submitted $2,386, a figure that none of its own calculations support." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392) as taxable SS instead of the lesser of the cap and the formula amount ($13,784.12), which overstated AGI at $51,672 and gave $4,021. It then submitted $1,858, which none of its own figures support." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Correctly computed taxable SS ($13,784), taxable income ($32,964) and tax of about $3,708 using the $12,400 10% bracket. It then submitted $2,534, which contradicts its own derivation." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Taxed the full 85% cap ($16,392.25) of SSDI without applying the lesser-of formula, which gives $13,784.12, and added the $284 tax-exempt pension to provisional income. It also fell back on the 2025 $15,000 deduction and the $11,925 10% bracket instead of 2026's $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Set up the correct taxable-SS formula but replaced its result with an 'approximation' of $15,650 instead of $13,784.12, and used the 2025 $11,925 10% bracket, reaching $3,941. It then submitted $1,611, which its own work does not support." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a projected $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100, and a $12,250 top for the 10% bracket instead of $12,400. This overstated taxable income at $33,664." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but subtracted a $15,300 standard deduction instead of the 2026 $16,100 and ended the 10% bracket at $12,200 instead of $12,400. This produced taxable income of $33,764 and tax of $3,808." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392.25) as taxable SS instead of the lesser formula amount of $13,784.12, and used a $15,400 deduction and a $12,225 10% bracket instead of 2026's $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA rules: an $8,100 standard deduction, a $5,300 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 deduction, no personal exemption, and 10%/12% brackets that break at $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a $15,700 standard deduction instead of $16,100 and the 2025 $11,925 10% bracket instead of 2026's $12,400. This overstated taxable income at $33,364 and the tax." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Applied the head-of-household, age-65-or-older standard deduction and claimed nonrefundable credits. The filer is a 56-year-old single person with no dependents, who gets only the $16,100 single deduction and has no credits." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Modeled 2026 under pre-TCJA rules, with a personal exemption and the smaller standard deduction. OBBBA made the TCJA structure permanent, so the correct calculation is the $16,100 deduction and 10%/12% brackets on $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but subtracted a pre-TCJA $8,300 standard deduction and a $5,150 personal exemption. Under 2026 law the deduction is $16,100 and there is no exemption, and the 10%/12% brackets apply." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 equals the tax when only 50% of the SSDI ($9,642.50) is taxable (AGI $44,922.50). Because provisional income is above the $34,000 threshold, the 85% tier makes $13,784.12 taxable." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064.13, but its $3,807.18 matches a $15,300 standard deduction with a $12,225 10% bracket. The 2026 values are $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation reaches $3,707.70 from AGI of $49,064.13, the 2026 single deduction and the 2026 brackets, but it submitted $3,744. The submitted value contradicts its own correct calculation." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a $15,350 standard deduction instead of the 2026 $16,100 and ended the 10% bracket at $12,200 instead of $12,400. This overstated taxable income at $33,714." -us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Added the $284 tax-exempt pension to provisional income, which raised taxable SS to $14,025.53 instead of $13,784.12. It also used the 2024 $14,600 deduction and $11,600 10% bracket instead of 2026's $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Applied only the 50% tier, 0.5 × ($44,922.50 − $25,000) = $9,961.25, and ignored the 85% tier for provisional income above $34,000, which makes $13,784.12 taxable. It also used a $15,750 deduction and a $12,000 10% bracket instead of $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction wipes out the tax. AGI is $49,064.12, so after the $16,100 deduction there is $32,964.12 of taxable income, taxed at 10%/12% with no nonrefundable credits." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Concluded that the deduction and credits reduce tax to $0. The $35,280 pension plus $13,784.12 of taxable SSDI leaves $32,964.12 taxable after the $16,100 deduction, and no nonrefundable credit applies." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used the 2025 $15,750 single standard deduction instead of the 2026 $16,100. This overstated taxable income by $350 ($33,314)." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $35,572 shows it taxed the 85% cap ($16,392) of SSDI instead of the lesser formula amount of $13,784.12, which overstated AGI by $2,608." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $33,206 is $241 above $32,964.12. That gap equals 0.85 × $284, so it added the tax-exempt pension to provisional income, which the taxable-SS formula does not count." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,736.66 implies taxable income of $33,205.50, which is $241.38 above $32,964.12. That gap equals 0.85 × $284, so it counted the tax-exempt pension in provisional income when computing taxable SS." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392) as taxable SS instead of the lesser of the cap and 0.85 × ($44,922.50 − $34,000) + $4,500 = $13,784.12, which overstated AGI at $51,672." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $2,444 implies taxable income of about $22,400, roughly $10,500 too low. That fits counting only a small part of the SSDI as taxable instead of the $13,784.12 required when provisional income is above $34,000." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but assumed the TCJA had expired: an $8,535 deduction, a $5,443 personal exemption and a 15% bracket. 2026 law keeps the $16,100 deduction, no exemption, and 10%/12% brackets." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but used a pre-TCJA $8,496 deduction plus a $5,419 personal exemption and a 15% second bracket. 2026 law has a $16,100 deduction, no exemption, and a 12% bracket above $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064, but treated 2026 as post-TCJA-sunset, with a $5,450 exemption, an $8,550 deduction and a 15% bracket. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent, so taxable income is $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Took the 85% cap ($16,392) as taxable SS instead of the lesser formula amount of $13,784.12, and used a $15,420 deduction and a $12,259 10% bracket instead of 2026's $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Gave no value or explanation for federal_income_tax_before_refundable_credits. -us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Got AGI right at $49,064.13, but applied a pre-TCJA 'deduction/exemption' and bracket structure that yields $4,708.37. 2026 law uses the $16,100 deduction with no personal exemption and 10%/12% brackets." -us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Left out the taxable SSDI ($13,784.12) and set AGI equal to the $35,280 pension alone. It also miscomputed the 10%-bracket tax as $1,540 instead of $1,240." -us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Added the $284 tax-exempt pension to provisional income and mis-added the total as $45,406 instead of $45,206.50. This inflated taxable SS to $14,196 instead of $13,784.12 and AGI to $49,476." -us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Used $6,000, the joint-filer amount, instead of the single filer's $4,500 in the 85%-tier formula, and added the $284 tax-exempt pension to provisional income, which gave $15,525.53 of taxable SS instead of $13,784.12. It also used a $15,750 deduction and a $12,150 10% bracket." -us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Left out the $13,784.12 of taxable SSDI, used a non-existent 12.5% rate and a $15,750 deduction to get $2,177, and then submitted $4,202, which its own calculation does not support." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $19,285 of SSDI as fully non-taxable, but provisional income of $44,922.50 is above the $34,000 threshold, so $13,784 of it is taxable. It also used the 2024 $14,600 standard deduction instead of the 2026 $16,100." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed taxable SSDI was about $0 because other income was 'low'. In fact $35,280 of pension plus half the SSDI gives provisional income of $44,922.50, which makes $13,784 taxable. Leaving that out cut taxable income to roughly $19,180." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It correctly got taxable SS of $13,784, AGI of $49,064 and taxable income of $32,964, and computed about $3,713. It then submitted $2,386 for an unexplained 'taxable SS recalculation' that contradicts its own derivation. It also used $12,150 as the top of the 10% bracket instead of the 2026 figure of $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It took the 85% cap ($16,392) as taxable SS instead of the lesser-of amount, 0.85 × $10,922.50 + $4,500 = $13,784, which overstated AGI at $51,672. It then submitted $1,858, a number unrelated to its own computed tax of $4,021." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its reasoning gets every step right (taxable SS $13,784, AGI $49,064, taxable income $32,964, tax about $3,708), but it submitted 2,534 as the value. The final output contradicts its own correct derivation." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Because provisional income was above $34,000, it taxed a flat 85% of SSDI ($16,392.25) instead of the lesser-of formula result of $13,784.12. It also added the $284 tax-exempt pension to provisional income. It then used 2025 parameters (a $15,000 standard deduction and a $11,925 10% bracket) instead of 2026's $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Instead of the formula result of $13,784, it used an unsupported 'approximation' of $15,650 for taxable SS and computed $3,941. It then submitted $1,611 with no supporting arithmetic, so the final value matches neither the correct liability nor its own calculation." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it used a $15,400 standard deduction instead of the 2026 OBBBA figure of $16,100, and a $12,250 10% bracket instead of $12,400, which overstated taxable income at $33,664." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it subtracted a $15,300 standard deduction instead of the 2026 single amount of $16,100, and used $12,200 instead of $12,400 as the top of the 10% bracket." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It counted the full 85% cap ($16,392.25) as taxable SS instead of the lesser-of amount ($13,784.12), which inflated AGI to $51,672. It also used a $15,400 standard deduction and a $12,225 10% bracket instead of 2026's $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA law: an $8,100 standard deduction, a $5,300 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, so the correct 2026 figures are a $16,100 standard deduction, no exemption, and 10%/12% brackets." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it used a $15,700 standard deduction instead of 2026's $16,100, and the 2025 $11,925 10% bracket instead of $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It applied a head-of-household standard deduction with the over-65 add-on. The head is 56 and files as single with no dependents, so only the $16,100 single deduction applies. The larger deduction drove its tax down to $1,391." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly used pre-TCJA rules (a smaller standard deduction plus a personal exemption, with a 15% bracket) for 2026. Under OBBBA the TCJA structure continues: a $16,100 single standard deduction, no exemption, and 10%/12% brackets." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it applied a pre-TCJA $8,300 standard deduction plus a $5,150 personal exemption and the 15% bracket, instead of the permanent post-OBBBA $16,100 standard deduction and 12% bracket." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"At 2026 brackets, $3,212 implies taxable income of about $28,830, roughly $4,130 below the correct $32,964. That points to under-including taxable SS (the correct amount is $13,784) or over-deducting relative to the $16,100 standard deduction." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064.13 is correct. However, $3,807.18 implies a standard deduction of about $15,350 and the 2025 $11,925 10% bracket, instead of the 2026 figures of $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation derives the correct $3,707.70 from AGI of $49,064.13, but the numeric value it submitted was $3,744. The final output contradicts its own correct computation." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it used a $15,350 standard deduction instead of 2026's $16,100, and a $12,200 10% bracket instead of $12,400, which overstated taxable income at $33,714." +us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $284 tax-exempt pension to provisional income, which raised taxable SS to $14,025.53. Only tax-exempt interest belongs in provisional income, so the correct amount is $13,784.12. It also used 2024 parameters (a $14,600 deduction and a $11,600 10% bracket) instead of 2026's $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It applied only the first-tier formula, 50% × (provisional income − $25,000) = $9,961, and ignored the second tier above $34,000. The second tier gives 0.85 × $10,922.50 + $4,500 = $13,784. It also used a $15,750 standard deduction and a $12,000 bracket instead of $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction offsets all income. In fact AGI of $49,064 ($35,280 pension plus $13,784 taxable SS) minus the $16,100 deduction leaves $32,964 of taxable income, so tax is $3,707.70, not zero." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It returned zero tax, as if deductions and credits eliminated all liability. The $35,280 pension plus $13,784 of taxable SS, minus the $16,100 standard deduction, leaves $32,964 taxable, and no nonrefundable credit applies." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 and its brackets are correct. However, it used the 2025 OBBBA single standard deduction of $15,750 instead of the 2026 inflation-adjusted $16,100, which overstated taxable income by $350." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $35,572 equals $51,672 − $16,100, which means it taxed the full 85% cap of SSDI ($16,392) instead of the lesser-of formula amount ($13,784)." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $33,206 is about $242 too high, which matches 0.85 × $284. In other words, it included the tax-exempt private pension in provisional income, but only tax-exempt interest counts." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"$3,736.66 corresponds exactly to taxable income of $33,205.53, which means it added the $284 tax-exempt pension to provisional income and raised taxable SS by $241.40. Only tax-exempt interest belongs in provisional income." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It set taxable SS at the 85% cap ($16,392) instead of the lesser-of amount, 0.85 × ($44,922.50 − $34,000) + $4,500 = $13,784, which inflated AGI to $51,672." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"At 2026 brackets, $2,444 implies taxable income of about $22,400, far below the correct $32,964. It understated the taxable portion of SSDI, which the 85% tier makes $13,784." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it assumed the TCJA had sunset and applied a projected $8,535 standard deduction, a $5,443 exemption and a 15% bracket, instead of OBBBA's permanent $16,100 deduction and 10%/12% brackets." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct. However, it applied post-TCJA-sunset parameters ($8,496 standard deduction, $5,419 exemption, 15% bracket) instead of the 2026 OBBBA $16,100 standard deduction and 12% bracket." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It explicitly assumed the TCJA had sunset for 2026, using an $8,550 deduction, a $5,450 exemption and a 15% bracket. OBBBA made the TCJA rates and the larger standard deduction permanent, so the correct deduction is $16,100 with no exemption, taxed at 10%/12%." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used the full 85% cap ($16,392) as taxable SS instead of the lesser-of amount ($13,784). It also used a $15,420 standard deduction and a $12,259 10% bracket instead of 2026's $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064.13 is correct. However, it applied a 'deduction/exemption' structure and higher rates consistent with pre-TCJA law, instead of the 2026 $16,100 standard deduction with 10%/12% brackets." +us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It left the $19,285 of SSDI out of AGI entirely, even though $13,784 is taxable once provisional income passes $34,000. It also miscomputed the 10% bracket tax as $1,540." +us,scenario_007,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 is correct, but it approximated the 2026 bracket and deduction figures instead of applying them exactly. The exact calculation is $16,100 deduction → $32,964.12 taxable → $1,240 + 12% × $20,564.12 = $3,707.70, not its $3,702." +us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added the $284 tax-exempt pension to provisional income and also added up its combined income wrong ($45,406 instead of $45,207). As a result it taxed $14,196 of SS instead of $13,784.12." +us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"In the second-tier formula it used $6,000, the married-filing-jointly figure, instead of the single filer's $4,500, and it also counted the tax-exempt pension in provisional income, giving taxable SS of $15,526. It also used a $15,750 deduction and a $12,150 bracket instead of 2026's $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left the $13,784 of taxable SSDI out of income, used a $15,750 deduction, and applied a non-existent 12.5% rate. It then submitted $4,202, which contradicts its own stated $2,177." us,scenario_007,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly noted that the head is under 65, but it assumed the 24-month SSDI waiting period was met because of the ""constant full-year status"" rule. A constant status over one tax year shows at most 12 months of receipt. No listed fact shows 24 months of entitlement, and the prompt treats unlisted status inputs as false, so neither the disability pathway nor the age pathway applies." -us,scenario_007,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model said SSDI income by itself confers Medicare eligibility and skipped the 24-month entitlement requirement for disability-based Medicare. No listed fact satisfies that requirement, and at 56 the head fails the age-65 test, so the answer is not eligible." -us,scenario_007,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model named the 24-month waiting period and then treated a year-long constant status as meeting it. One constant year gives at most 12 months, and the prompt treats unlisted facts, including how long SSDI has been received, as false or 0. The 24-month test fails, and the head at 56 fails the age-65 test." -us,scenario_007,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model stated that Medicare follows after 24 months of SSDI receipt but never checked whether this household met that duration. It simply assumed it did. Nothing listed shows 24 months of entitlement and unlisted inputs default to 0 or false, so at 56 the head is not Medicare eligible." -us,scenario_007,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine treats SSDI recipients as Medicare eligible once ""the qualifying period"" has passed, and assumed that period had passed without any listed fact to support it. The 24-month entitlement requirement is not established, and the head is under 65, so the reference returns not eligible." -us,scenario_007,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model reasoned that because SSDI is treated as constant through the year, the head had already passed the 24-month waiting period. Constancy within one tax year covers only 12 months, and the prompt forbids inferring unlisted status facts. The disability pathway is unmet, and at 56 the head fails the age-65 test." -us,scenario_007,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated receipt of SSDI income as ""indicating"" Medicare eligibility and ignored the 24-month entitlement requirement for disability-based Medicare. No listed fact satisfies that requirement, and the head is 56, below the age-65 threshold." -us,scenario_007,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model said SSDI benefits confer Medicare eligibility under age 65 without applying the 24-month entitlement wait. Duration of receipt is an unlisted input that defaults to 0 or false, so the disability pathway fails and the head at 56 is not eligible." -us,scenario_007,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model equated being an SSDI recipient with being Medicare eligible and skipped the 24-month entitlement requirement. The household facts do not establish that requirement, and the head is 56, so neither the age pathway nor the disability pathway applies." -us,scenario_007,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model said SSDI receipt qualifies the head for Medicare and never applied the 24-month waiting period. No listed fact meets that period and unlisted facts default to false, so at 56 the head is not Medicare eligible." -us,scenario_007,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model invoked ""disability rules"" but treated current SSDI income as enough, leaving out the 24-month entitlement condition in those same rules. That condition is not established, and at 56 the head fails the age-65 test." -us,scenario_007,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model concluded that SSDI receipt qualifies the head for Medicare without testing the 24-month entitlement requirement. The facts do not establish it, and the head is under 65, so the answer is not eligible." -us,scenario_007,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model asserted that PolicyEngine grants Medicare to anyone receiving SSDI, basing this on the $19,285 of income alone. Disability-based Medicare requires 24 months of SSDI entitlement, which no listed fact establishes, and at 56 the head fails the age-65 test." -us,scenario_007,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly failed the head on the age test at 56, then asserted that PolicyEngine sets Medicare eligibility to yes whenever SSDI income is present. The disability pathway requires 24 months of SSDI entitlement, and no disability status or duration of receipt is listed, so the pathway is not met." -us,scenario_007,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model accepted that the head is under 65 but treated SSDI income as enough for disability-based Medicare. It left out the 24-month entitlement requirement, which the facts do not establish because unlisted status and duration inputs are false or 0." -us,scenario_007,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model applied a disability-based Medicare rule keyed only on receiving SSDI income and skipped that rule's 24-month entitlement condition. No listed fact meets that condition, and the head at 56 fails the age-65 test, so the correct answer is not eligible." -us,scenario_007,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model claimed the modeled disability pathway turns on SSDI benefits being positive. The disability pathway actually requires 24 months of SSDI entitlement, and positive current-year income does not establish that. With no such fact listed and the head at 56, the head is not Medicare eligible." -us,scenario_007,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model applied a ""modeled eligibility rule"" that grants disability-based Medicare to anyone receiving SSDI and left out the 24-month entitlement requirement. That requirement is unmet on the listed facts, and the head is under 65, so the answer is not eligible." -us,scenario_007,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model asserted that SSDI income qualifies the head for Medicare under PolicyEngine rules and skipped the 24-month entitlement requirement for the disability pathway. That requirement is not established, and the head at 56 fails the age-65 test." -us,scenario_007,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"The model cited the 24-month waiting period and then assumed it had passed with no supporting fact. The prompt says to treat unlisted facts, including how long SSDI has been received, as 0 or false. The waiting period is unmet, and the head at 56 is not Medicare eligible." -us,scenario_007,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model applied disability-based Medicare because SSDI income is present and ignored the 24-month entitlement condition that pathway requires. The listed facts do not establish that condition, so the head, who is under 65, is not eligible." -us,scenario_007,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model treated SSDI income alone as establishing disability-based Medicare and never applied the 24-month entitlement requirement. That requirement is unmet on the listed facts, and the head at 56 fails the age-65 test." +us,scenario_007,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"Correctly noted the head is under 65 and that SSDI-based Medicare needs a 24-month waiting period, but then assumed that period was met because status is held constant for the year. Being constant within the year says nothing about 24 prior months. The months-of-SSDI input is unlisted, so it is 0 and the waiting-period test fails." +us,scenario_007,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated receipt of SSDI income as automatically giving Medicare eligibility and skipped the 24-month entitlement requirement. With zero recorded months of SSDI and the head aged 56, neither the disability pathway nor the age-65 pathway applies." +us,scenario_007,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Acknowledged the 24-month SSDI waiting period but read the year-long constant-status instruction as meeting it. That instruction covers only the tax year. The unlisted months of SSDI receipt default to 0, so the head, at 56, is not Medicare eligible." +us,scenario_007,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Correctly stated that Medicare follows 24 months of SSDI receipt, then assumed those 24 months had happened even though the household facts give no duration. The months-of-SSDI input defaults to 0, so the head, at 56, fails both the waiting-period test and the age-65 test." +us,scenario_007,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"Assumed the qualifying period was met and claimed PolicyEngine treats any SSDI recipient as Medicare eligible. PolicyEngine requires 24 months of SSDI receipt, and that input is unlisted, so it is 0. A 56-year-old with no disability or ESRD flag is not eligible." +us,scenario_007,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Stated the 24-month waiting rule and then assumed it was satisfied because disability income is constant through the year. Constancy within one tax year does not establish 24 months of prior entitlement. The unlisted months-of-SSDI input is 0, so the 56-year-old head is not Medicare eligible." +us,scenario_007,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Read a positive SSDI income amount as proof of Medicare eligibility and ignored the 24-month entitlement requirement. Months of SSDI receipt are unlisted and therefore 0, and the head is 56, so no Medicare pathway is met." +us,scenario_007,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Assumed SSDI receipt gives Medicare eligibility under 65 without applying the 24-month waiting period. With zero recorded months of SSDI and no ESRD or disability flag, the 56-year-old head is not eligible." +us,scenario_007,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Equated being an SSDI recipient with being Medicare eligible and never applied the 24-month entitlement test. That test fails because months of SSDI receipt default to 0, and the head is 56." +us,scenario_007,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Treated SSDI income alone as qualifying for Medicare and omitted the 24-month waiting period. The months-of-SSDI input is unlisted and defaults to 0, so the 56-year-old head does not qualify." +us,scenario_007,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Invoked the disability rules without their 24-month SSDI entitlement requirement. That requirement is unmet because months of receipt default to 0, so the 56-year-old head is not Medicare eligible." +us,scenario_007,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Concluded that SSDI receipt qualifies the head for Medicare without checking the 24-month waiting period. The months-of-SSDI input is 0, so with the head at 56 neither the disability pathway nor the age pathway is satisfied." +us,scenario_007,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"Claimed PolicyEngine's Medicare rule covers any SSDI recipient, leaving out the 24-month entitlement condition. PolicyEngine requires 24 months of SSDI receipt, the unlisted months input is 0, and the 56-year-old head is ineligible." +us,scenario_007,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Correctly rejected the age pathway at 56, then assumed PolicyEngine's disability pathway triggers on any SSDI income. That pathway requires 24 months of SSDI receipt, which defaults to 0 here, and the disability flag is unset, so the head is not eligible." +us,scenario_007,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"Treated positive SSDI income as enough for disability-based Medicare and ignored the 24-month waiting period PolicyEngine enforces. The months-of-SSDI input is unlisted and therefore 0, so the under-65 head is ineligible." +us,scenario_007,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Applied a disability-based Medicare rule keyed only to SSDI receipt and skipped its 24-month entitlement requirement. With zero recorded months of SSDI, the 56-year-old head does not qualify." +us,scenario_007,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"Assumed the modeled disability pathway turns on whenever SSDI benefits are positive. PolicyEngine's pathway requires at least 24 months of SSDI receipt, that input defaults to 0, and the 56-year-old head is not eligible." +us,scenario_007,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"Misstated the modeled rule as SSDI receipt alone. The disability pathway requires 24 months of SSDI entitlement, the unlisted months input is 0, and the head is under 65, so Medicare eligibility is No." +us,scenario_007,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"Treated SSDI income as directly qualifying for Medicare and ignored the 24-month waiting period. Months of SSDI receipt default to 0, so the 56-year-old head fails both Medicare pathways." +us,scenario_007,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"Correctly stated the rule that Medicare follows 24 months of SSDI, then applied it as if it were met with no supporting fact. Months of SSDI receipt are unlisted, so they are 0. The 24-month condition fails and the 56-year-old head is not eligible." +us,scenario_007,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,Applied disability-based Medicare under 65 using SSDI receipt alone and omitted the 24-month entitlement requirement. That requirement is unmet because the months-of-SSDI input defaults to 0. +us,scenario_007,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disability,False,"Read full-year SSDI receipt as meeting the disability pathway. That pathway requires 24 months of SSDI entitlement, and one year of constant status does not supply it. The months input is 0, so the 56-year-old head is ineligible." +us,scenario_007,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"Concluded that SSDI receipt alone qualifies the head for Medicare without applying the 24-month waiting period. With zero recorded months of SSDI and the head aged 56, Medicare eligibility is No." us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model first recognized that no wage was listed but then asserted $1,516.30 of employee Social Security and Medicare tax on unspecified “applicable taxable wages.” The listed Social Security disability and private pension payments are not employee FICA wages, and the 50 weekly hours cannot create wage income when the unlisted wage rate and annual wages are zero." us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." us,scenario_007,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"It correctly got Idaho taxable income of $19,180. It then applied the old 5.695% rate instead of 2026's 5.3% and left out the $4,920 zero-rate amount. After computing about $1,092, it raised the result to $1,571 for no stated reason, when 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It correctly derived $19,180 of taxable income and the 5.3% rate. However, it taxed the whole $19,180 and left out Idaho's $4,920 single-filer zero-rate amount, which would have taken $260.76 off the tax." -us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It made up a $41,110 retirement-income exclusion for private pensions. Idaho's retirement-benefits deduction covers only certain public, military, and similar pensions, and only from age 65 (62 if disabled), so it does not apply to a 56-year-old's private pension. The $35,280 pension stays taxable, giving $19,180 of taxable income after the $16,100 standard deduction and $755.78 of tax." -us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,state_local_rule,False,"Its $0 answer treats the pension as fully sheltered, but no Idaho deduction offsets it. Idaho income of $35,280 minus the $16,100 standard deduction leaves $19,180, and 5.3% on the amount above $4,920 is $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It correctly reached $19,180 of Idaho taxable income. It then used 5.695% instead of 5.3%, left out the $4,920 zero-rate amount, and raised its own $1,092 figure to $1,379 based on an unexplained 'conformity estimate'." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It used a $15,750 standard deduction instead of 2026's $16,100, the 5.695% rate instead of 5.3%, and a $4,673 zero-rate amount instead of $4,920. It then raised its own $846 result to $1,300 without any computation." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It stated the right method: 5.3% applied to $19,180 above roughly $4,700. However, the $1,112 it submitted equals 5.8% of the full $19,180, with no zero-rate amount and the wrong rate. Applying its own stated method with the correct $4,920 zero-rate amount gives $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It used 2025's $4,811 zero-rate amount instead of 2026's inflation-adjusted $4,920. That taxed an extra $109 at 5.3%, giving $761.56 instead of $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It wrongly decided Idaho has no Social Security subtraction and taxed $16,392 of disability benefits. Idaho subtracts all federally taxable Social Security, which here is $13,784 under the provisional-income formula, not a flat 85%. It also made up a $4,950 personal exemption, used a $15,000 standard deduction and the 5.695% rate, and subtracted the grocery credit, which Idaho pays as a refundable credit." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It used a $14,600 standard deduction instead of $16,100 and 5.695% instead of 5.3%, and left out the $4,920 zero-rate amount. It then raised its own $1,178 estimate to $1,800 for no reason." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It correctly got $19,180 of taxable income and the 5.3% rate. It then applied 5.3% to the whole amount and left out the $4,920 single-filer zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It never applied Idaho's subtraction of the $13,784 of federally taxable Social Security and taxed the full $49,064 federal AGI. It also used a $14,000 standard deduction instead of $16,100 and 5.695% instead of 5.3%, and left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It correctly subtracted Social Security. It then used a $15,300 standard deduction instead of $16,100 and the old 5.8% rate instead of 5.3%, and taxed the whole amount without the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It correctly started from $35,280. It then used a $15,400 standard deduction instead of $16,100 and 5.695% instead of 5.3%, and left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,100 standard deduction instead of 2026's $16,100, which overstated taxable income by $8,000. It also applied 5.3% to the whole amount without the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"It used a $15,700 standard deduction instead of $16,100 and the old 5.8% rate instead of 5.3%, and taxed the whole amount without Idaho's $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It counted 'taxable disability income' in Idaho income even though Idaho subtracts all federally taxable Social Security. Its $1,582 answer corresponds to a taxable base of roughly $28,000–$30,000, not $19,180. It also left out the $4,920 zero-rate amount on top of 5.3%." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It correctly excluded Social Security. However, its $1,246 answer matches a flat rate (5.695% or 5.8%) applied to about $21,500–$21,900 with no zero-rate amount, which means the standard deduction was too small and the $4,920 zero-rate amount was missing. The correct result is 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It took a combined $13,450 standard deduction and personal exemption, but Idaho has no personal exemption and its 2026 standard deduction is $16,100. That gave $21,830 of taxable income, which it taxed at 5.695% instead of 5.3%, without the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"Its $3,371 answer requires a taxable base above the household's entire $54,565 gross income at any Idaho rate. That means it neither subtracted Social Security nor applied the standard deduction or the zero-rate amount. The correct base is $19,180, taxed at 5.3% above $4,920." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It correctly got Idaho AGI of $35,280. Its $1,115.08 equals 5.695% of $19,580, so it used a $15,700 standard deduction instead of $16,100, the old 5.695% rate instead of 5.3%, and no $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation computes $1,092.30, which is 5.695% of $19,180, with the wrong rate and no $4,920 zero-rate amount. It then submitted $1,894, which contradicts its own explanation. The correct figure is 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It used a $15,350 standard deduction instead of $16,100 and the old 5.8% rate instead of 5.3%, and taxed the whole $19,930 without the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It applied Idaho's retirement-benefits deduction to the $35,280 private pension. That deduction covers only qualifying public, military, and similar pensions, and only for filers aged 65 or older (62 if disabled), and this head is 56. Without it, taxable income is $19,180 and the tax is $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It used a $15,750 standard deduction instead of $16,100, which gave $19,530 instead of $19,180. It then applied 5.3% to the full amount without the $4,920 zero-rate amount and added a $10 permanent building fund tax, which is not part of this output." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It claimed the standard deduction fully offsets income, but the $35,280 taxable pension is well above the $16,100 standard deduction plus the $4,920 zero-rate amount. That leaves $14,260 taxed at 5.3%." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It returned $0 without showing any calculation. Idaho income of $35,280 minus the $16,100 standard deduction leaves $19,180, and 5.3% on the amount above $4,920 is $755.78, so the pension income is taxable." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It used a $5,000 zero-rate amount instead of 2026's $4,920. That undertaxed $80 at 5.3% and gave $751.54 instead of $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"Its $1,885 equals 5.3% of about $35,570, which is federal AGI with 85% of Social Security included ($51,672) minus $16,100. That means it never applied Idaho's full subtraction of taxable Social Security, and it also left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It correctly derived $19,180 of taxable income and the 5.3% rate. It then taxed the full amount and left out Idaho's $4,920 single-filer zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"Its $1,759.89 equals 5.3% of about $33,205, which is federal taxable income (federal AGI minus $16,100). That means it left out Idaho's subtraction of the $13,784 of federally taxable Social Security as well as the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It subtracted $2,080 of health premiums that do not reduce Idaho taxable income here. Idaho AGI of $35,280 minus only the $16,100 standard deduction gives $19,180. It also used 2025's $4,811 zero-rate amount instead of 2026's $4,920." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"Its $35,572 taxable income is $51,672 of federal AGI (with 85% of Social Security included) minus $16,100, so it never applied Idaho's subtraction of federally taxable Social Security. It also used a flat 85% inclusion instead of the $13,784 formula amount and left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It used an approximate $4,990 zero-rate amount instead of 2026's $4,920. That undertaxed $70 at 5.3% and gave $752 instead of $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It carried forward 2025's $4,811 zero-rate amount instead of 2026's inflation-adjusted $4,920. That taxed an extra $109 at 5.3% and gave $761.56." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"Its $1,477 equals 5.3% of about $27,870, which means a standard deduction far below 2026's $16,100 and no $4,920 zero-rate amount. The correct calculation is 0.053 × ($35,280 − $16,100 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It used a pre-TCJA-style $8,535 standard deduction instead of 2026's $16,100, which the One Big Beautiful Bill Act made permanent. It also applied the old 5.695% rate instead of 5.3% and left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It used a roughly $8,480 'conforming' standard deduction instead of $16,100, which gave $26,803 instead of $19,180. It then applied 5.695% instead of 5.3% and left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It assumed the TCJA standard deduction expired and used the post-sunset $8,550. The One Big Beautiful Bill Act made the higher deduction permanent, and it is $16,100 for 2026. It also applied 5.3% to the full $26,730 without the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"Its $36,252 base is federal AGI with 85% of Social Security included ($51,672) minus $15,420, so it never applied Idaho's subtraction of taxable Social Security. It also used the old 5.8% rate and left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It taxed the federal taxable income of $32,964 directly. That skips Idaho's subtraction of the $13,784 of federally taxable Social Security, which brings the base down to $19,180, and it also left out the $4,920 zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score." -us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It used a $13,600 deduction instead of $16,100 and a $2,500 zero-rate amount instead of $4,920. These errors offset each other enough that 5.3% ended up applied to the whole $19,180, overstating tax by the $4,920 zero-rate portion." -us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It claimed Idaho has no individual income tax. Idaho taxes individual income at a flat 5.3% above a $4,920 zero-rate amount, which here yields $755.78 on $19,180 of taxable income." -us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It correctly got $19,180 of taxable income and the 5.3% rate. It then applied the rate to the full amount without Idaho's $4,920 single-filer zero-rate amount." -us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"It treated all $19,285 of Social Security disability benefits as taxable and never applied Idaho's subtraction of federally taxable Social Security. It also used a $14,600 standard deduction and the 5.8% rate, subtracted the grocery credit (refundable in Idaho), and then cut the result to $1,707.48 for no stated reason." -us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It took a second $5,200 standard deduction from a base that had already been reduced by the federal deduction. It also applied graduated brackets that Idaho repealed in 2023. The correct calculation applies the single flat 5.3% rate above $4,920 to $19,180." +us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It reached $19,180 of taxable pension income but taxed it at the outdated 5.695% rate with no $4,811 zero-rate band. It never subtracted the $2,080 of health insurance premiums and left out the $10 permanent building fund tax. It then raised its own $1,092 result to $1,571 without basis; the correct figure is 5.3% of ($17,100 − $4,811) + $10 = $661.32." +us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the taxable Social Security and the $16,100 standard deduction. It skipped Idaho's $2,080 health insurance premium subtraction, so taxable income was $19,180 instead of $17,100. It applied 5.3% from the first dollar instead of above the $4,811 zero-rate band and left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"It applied Idaho's retirement-benefits deduction to wipe out the $35,280 private pension. That deduction covers only qualifying public, civil-service and military retirement pay, starting at age 65 (or 62 if disabled), so it does not apply to a 56-year-old's private pension. The pension stays taxable, and after the Social Security and health-premium subtractions and the $16,100 deduction, Idaho taxable income is $17,100." +us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It said there was no Idaho liability without doing a calculation. The $35,280 taxable pension minus the $2,080 health-premium subtraction and the $16,100 standard deduction leaves $17,100 of taxable income. That income owes 5.3% above $4,811 plus the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached $19,180 but used the outdated 5.695% rate instead of 5.3% above a $4,811 zero-rate band. It never took the $2,080 health insurance premium subtraction or added the $10 building fund tax. It then raised its own $1,092 to $1,379 for no reason." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,750 instead of $16,100 for 2026, skipped the $2,080 health-premium subtraction, and used the 5.695% rate with a made-up ~$4,673 threshold. It then raised its own $846 to $1,300 without basis. The correct path is 5.3% of ($17,100 − $4,811) + $10." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the $2,080 health insurance premium subtraction, so it started from $19,180 instead of $17,100. Its own method (5.3% above about $4,700) gives roughly $767, so the $1,112 it submitted does not follow from its stated steps. It also left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It correctly applied 5.3% above the $4,811 zero band but never subtracted the $2,080 of health insurance premiums, so it taxed $19,180 instead of $17,100. It also left out the $10 permanent building fund tax, which is part of Idaho tax before refundable credits." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly said Idaho has no Social Security subtraction and taxed 85% of the benefits. It also used a $15,000 deduction instead of $16,100, a $4,950 personal exemption that Idaho does not have, and the 5.695% rate. It then subtracted the grocery credit, which is refundable and does not reduce tax before refundable credits." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $14,600 standard deduction instead of $16,100, the 5.695% rate instead of 5.3% above $4,811, and no health-premium subtraction. It then raised its own $1,178 to $1,800 with an unsupported 'phase-in' adjustment." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It taxed $19,180 at a flat 5.3% from the first dollar. It missed Idaho's $2,080 subtraction for health insurance premiums (taxable income is $17,100) and the $4,811 zero-rate band, and it left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It started from the full $49,064 federal AGI and never took Idaho's subtraction for the $13,784 of taxable Social Security. It also used a $14,000 deduction instead of $16,100, skipped the $2,080 health-premium subtraction, and taxed at 5.695% instead of 5.3% above $4,811." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of $16,100 and the repealed 5.8% rate instead of 5.3% above the $4,811 zero-rate band. It never took the $2,080 health insurance premium subtraction or added the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of $16,100 and a flat 5.695% instead of 5.3% above $4,811. It skipped the $2,080 health-premium subtraction (correct taxable income is $17,100) and the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used an $8,100 standard deduction instead of Idaho's federally conforming $16,100, which overstated taxable income at $27,180. It also missed the $2,080 health-premium subtraction, the $4,811 zero-rate band and the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a $15,700 standard deduction instead of $16,100 and the superseded 5.8% rate instead of 5.3% above $4,811. It never applied the $2,080 health insurance premium subtraction or added the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It treated the disability benefits as Idaho-taxable, but Idaho fully subtracts the $13,784 of Social Security included in federal AGI. Its $1,582 is consistent with a flat rate on roughly $28,000–30,000 of income. It applied neither the $2,080 health-premium subtraction nor the $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,246 matches a flat 5.695% on about $21,900, so it used an outdated rate and too small a deduction. The correct base is $17,100 after the $2,080 health-premium subtraction and $16,100 deduction, taxed at 5.3% above $4,811, plus the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took about $13,450 of 'standard deduction and personal exemption' instead of the $16,100 standard deduction, and Idaho has no personal exemption. It also skipped the $2,080 health-premium subtraction and taxed $21,830 at 5.695% instead of 5.3% above $4,811." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $3,371 matches a flat rate on about $59,000–64,000, which is more than the household's total income. It never subtracted Social Security, the $2,080 of health premiums, or the $16,100 standard deduction from the $35,280 pension." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $1,115.08 equals ($35,280 − $15,700) × 5.695%. It used a too-small $15,700 deduction and the outdated 5.695% flat rate instead of 5.3% above $4,811, and it missed the $2,080 health insurance premium subtraction." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its explanation computes $1,092.30 (= $19,180 × 5.695%) but it submitted $1,894. Both figures are wrong: it used the outdated 5.695% rate with no $4,811 zero-rate band and missed the $2,080 health-premium subtraction and the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of $16,100 and the repealed 5.8% flat rate instead of 5.3% above $4,811. It skipped the $2,080 health-premium subtraction and the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,age_disability,False,"It used Idaho's retirement-benefits deduction to wipe out the $35,280 private pension. That deduction covers only qualifying public and military retirement pay for filers 65 and over (or 62 and disabled), so it does not reach a 56-year-old's private pension. Idaho taxable income stays at $17,100." +us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It correctly added the $10 permanent building fund tax but used the 2025 $15,750 deduction instead of the 2026 $16,100. It also skipped the $2,080 health insurance premium subtraction and taxed $19,530 at 5.3% from the first dollar instead of above the $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction cancels all Idaho income, but the $35,280 taxable pension is well above the $16,100 deduction plus the $2,080 health-premium subtraction. That leaves $17,100 taxed at 5.3% above $4,811, plus the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It returned $0 without a derivation. The $35,280 taxable pension minus the $2,080 health-premium subtraction and the $16,100 standard deduction leaves $17,100 taxable, and no nonrefundable credit removes that tax." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $5,000 zero bracket instead of $4,811 and never subtracted the $2,080 of health insurance premiums, so it taxed $14,180 instead of $12,289. It also left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $1,885 equals 5.3% of about $35,572, which is federal AGI with 85% of benefits taxable ($51,672) minus $16,100. It never took Idaho's subtraction for taxable Social Security, the $2,080 health-premium subtraction, or the $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It taxed $19,180 at a flat 5.3% from the first dollar. It missed the $2,080 health insurance premium subtraction (taxable income is $17,100), the $4,811 zero-rate band and the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $1,759.89 equals 5.3% of $33,205, which is federal taxable income with the taxable Social Security still in it. It skipped Idaho's full Social Security subtraction, the $2,080 health-premium subtraction and the $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,other,False,"It computed the $651.32 of income tax correctly, after the Social Security and health-premium subtractions and 5.3% above $4,811. It left out Idaho's $10 permanent building fund tax, which this household owes and which is part of state income tax before refundable credits." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Its $35,572 taxable income is federal AGI with 85% of benefits taxable ($51,672) minus $16,100. It never subtracted Social Security, which Idaho excludes entirely, or the $2,080 of health premiums, and it applied 5.3% with no $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It used a ~$4,990 zero-rate amount instead of $4,811 and never subtracted the $2,080 of health insurance premiums, so it taxed $14,190 instead of $12,289. It also left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It correctly applied 5.3% above $4,811 but missed Idaho's $2,080 health insurance premium subtraction, so it taxed $19,180 instead of $17,100. It also left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $1,477 matches a flat rate on about $26,000–28,000, which means it used far less than the $16,100 standard deduction. It also skipped the $2,080 health-premium subtraction and the $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a pre-TCJA-style $8,535 deduction instead of the permanent $16,100 that Idaho follows. It also taxed at 5.695% instead of 5.3% above $4,811 and skipped the $2,080 health-premium subtraction." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a roughly $8,500 'conforming' deduction instead of $16,100, which gave $26,803 of taxable income. It taxed that at 5.695% instead of 5.3% above $4,811 and missed the $2,080 health-premium subtraction." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA standard deduction would expire and used $8,550, but federal law made the higher deduction permanent ($16,100 for 2026) and Idaho follows it. It also missed the $2,080 health-premium subtraction, the $4,811 zero-rate band and the $10 building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Its $36,252 is federal AGI with 85% of benefits taxable ($51,672) minus a $15,420 deduction. It skipped Idaho's Social Security subtraction and the $2,080 health-premium subtraction, and it used the repealed 5.8% rate instead of 5.3% above $4,811." +us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It used federal taxable income of $32,964 ($49,064 − $16,100) as the Idaho base and never removed the $13,784 of taxable Social Security, which Idaho subtracts in full. It also missed the $2,080 health-premium subtraction and applied 5.3% with no $4,811 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used a $2,500 zero bracket instead of $4,811 and never subtracted the $2,080 of health insurance premiums, so it taxed $19,180 instead of $12,289. It also left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It claimed Idaho has no individual income tax, which is false. Idaho taxes income at 5.3% above a $4,811 zero-rate band and adds a $10 permanent building fund tax, so this household owes tax on $17,100 of taxable income." +us,scenario_007,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It taxed $19,180 at a flat 5.3% from the first dollar. It missed the $2,080 health insurance premium subtraction (taxable income is $17,100), the $4,811 zero-rate band and the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It reached $19,180 after the Social Security subtraction but skipped Idaho's $2,080 health insurance premium subtraction. It applied 5.3% from the first dollar instead of above $4,811 and left out the $10 permanent building fund tax." +us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It counted 100% of the disability benefits in federal AGI and did not subtract Social Security, which Idaho excludes. It used a $14,600 deduction and the repealed 5.8% rate, and it subtracted the refundable grocery credit. It then lowered its own $2,198 to $1,707 without basis." +us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It took a second, made-up $5,200 Idaho deduction on top of federal taxable income and applied invented graduated brackets. Idaho has one 5.3% rate above a $4,811 zero-rate band, applied to $17,100 after the $2,080 health-premium subtraction, plus the $10 building fund tax." us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Correctly picked the refundable Idaho Grocery Credit as the only refundable state credit, but used the old $120 per-person amount. HB 231 raised the under-65 amount to $155 starting in tax year 2025, so the 2026 credit is $155." us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Said Idaho has no refundable individual income tax credits and missed the Idaho Grocery Credit, which is refundable. That credit gives this 12-month-qualifying resident $155 for 2026." us,scenario_007,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,Said no refundable state credits apply and missed the refundable Idaho Grocery Credit. That credit is $155 per person for 2026 for a full-year resident who is not on SNAP. @@ -538,48 +546,49 @@ us,scenario_008,child6_early_head_start_eligible,gpt-5.4-mini,llm_error,categori us,scenario_008,child6_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 1 as within the preschool-age Head Start range. A one-year-old falls under Early Head Start rather than Head Start, and age alone does not establish income eligibility." us,scenario_008,child6_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact and failed to apply the age-based YOUNG_CHILD pathway. Child 6 is age 1 and a dependent, and the household's MAGI of 0.55 times FPL satisfies the New Jersey income test for that category." us,scenario_008,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It applied an EITC phase-out starting near $30,050, but the 2026 MFJ threshold is above AGI, so the full $8,231 applies. It also computed the ACTC from earned income of $30,494 (0.9235×SE) instead of $30,517.41 (SE income minus half of SE tax). Finally, it submitted $12,130, which is below even its own $12,372 sum." -us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It put the EITC at about $1,395. A married couple with three or more children and about $30.5k of earned income is on the plateau and gets the full $8,231. It also computed an ACTC of $4,245 but then used an unexplained $1,305; the correct figure is $4,202.61." -us,scenario_008,federal_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It acknowledged the EITC was positive and still reported $0. That drops the $8,231 maximum EITC and the $4,202.61 refundable CTC (15% of the $28,017.41 of earnings above $2,500), both of which are paid regardless of tax liability." -us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its ACTC of $4,202 was right, but it overstated the 2026 EITC maximum as $8,267 instead of $8,231. It then inflated its own $12,469 total to an unsupported $13,167, even though no phase-out or other credit applies." -us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It added a $1,000 refundable AOTC even though qualified education expenses are $0, so the AOTC is $0. It also cut the ACTC to about $388 for supposed nonrefundable use. With zero tax liability, the full $4,202.61 (15% of earned income above $2,500) is refundable." -us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It reduced the EITC to about $7,700, but AGI is below the 2026 MFJ phase-out threshold, so the full $8,231 applies. It also used gross $30,800 earnings for the ACTC instead of $30,517.41 after the half-SE-tax deduction, and it added a $355 AOTC even though there are no qualified expenses." -us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used 2025 EITC parameters with a $26,511 MFJ phase-out start, which reduced an $8,046 maximum to $7,178. The 2026 maximum is $8,231, and the 2026 MFJ phase-out threshold is above AGI, so there is no reduction. It also computed the ACTC on $30,494 of earned income instead of $30,517.41." -us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It phased the EITC down to $7,856 from a ~$29,600 threshold instead of using the unreduced 2026 maximum of $8,231. It treated the ACTC as up to $8,500 even though the 15% earned-income formula limits it to $4,202.61. It also added a $1,000 AOTC with $0 qualified expenses and settled on an unsupported $12,900." -us,scenario_008,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It understated the EITC at $8,021 instead of the full $8,231 2026 maximum. It also computed the ACTC on gross $30,800 earnings. Earned income must reflect the half-SE-tax deduction ($30,517.41), which gives $4,202.61 rather than $4,245." -us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It phased the EITC down to $7,787 instead of using the full $8,231 maximum, since AGI is below the 2026 MFJ threshold. It also used the expired pre-TCJA $3,000 ACTC earnings threshold instead of the permanent $2,500 threshold." -us,scenario_008,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $4,124 matches 15% × ($30,494 − $3,000), which uses the expired $3,000 threshold instead of $2,500 and omits the half-SE-tax adjustment to earned income. Its EITC of $8,190 is also below the $8,231 2026 maximum." -us,scenario_008,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,247 instead of $8,231. It also counted SE earned income as 0.9235×$4,000 = $3,694 instead of $4,000 minus half of SE tax ($3,717.41). That gives an ACTC of $4,199.10 instead of $4,202.61." -us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It phased the EITC down to $7,691, but AGI is below the 2026 MFJ phase-out threshold, so the full $8,231 applies. Its ACTC used $30,494 of earned income instead of $30,517.41." -us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said the household gets the maximum EITC plus the CTC but reported $11,400, which is inconsistent with that. The maximum EITC is $8,231 and the refundable CTC is $4,202.61, which together give $12,433.61." -us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its ACTC of $4,124 reflects the expired $3,000 earnings threshold applied to $30,494. The correct figure is 15% × ($30,517.41 − $2,500) = $4,202.61. It also rounded the EITC to $8,200 instead of the $8,231 maximum." -us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a phase-out that cut the EITC to $7,128, but AGI is below the 2026 MFJ threshold, so the full $8,231 applies. It also computed the ACTC as $4,128 using the expired $3,000 threshold instead of $2,500." -us,scenario_008,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no breakdown, and $9,750 is about $2,700 short. The correct total is the full $8,231 2026 EITC plus the $4,202.61 refundable CTC, so the answer understates one or both credits." -us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,203 was correct, but it used the 2025 three-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231, which is exactly the $185 gap." -us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no component breakdown, and $11,940 is about $494 short. That shortfall implies a phased-down or outdated EITC instead of the full $8,231 2026 maximum, added to the $4,202.61 refundable CTC." -us,scenario_008,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,128 uses the expired $3,000 earnings threshold instead of $2,500. The correct figure is 15% × ($30,517.41 − $2,500) = $4,202.61. Its EITC of $8,233 is also $2 above the $8,231 maximum." -us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no answer or explanation for federal_refundable_credits, so no value could be scored." -us,scenario_008,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,360 instead of $8,231. It also computed the ACTC from $30,494 of earned income (0.9235×SE) instead of $30,517.41 (SE income minus half of SE tax)." -us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It wrongly concluded that the qualifying children were not established. Six resident children aged 1–17 qualify for the three-or-more-child EITC ($8,231), and the five under 17 generate a refundable CTC of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It reported $0 even though the household has $30,517.41 of earned income and six qualifying children. That income produces the full $8,231 EITC and a refundable CTC of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but it computed the ACTC on gross $30,800 of earnings. Self-employment earned income must be reduced by the half-SE-tax deduction ($282.59), which makes earned income $30,517.41 and the ACTC $4,202.61 rather than $4,245." -us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the 2025 EITC maximum of $8,046 instead of the 2026 maximum of $8,231. It also computed the ACTC on gross $30,800 instead of $30,517.41 after the half-SE-tax deduction, which gives $4,245 instead of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but its ACTC of $4,245 treats the full $4,000 of SE income plus wages as earned income. Earned income nets out half of SE tax, giving $30,517.41 and an ACTC of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 equals $8,231 + $4,245, which means it computed the refundable CTC on gross $30,800 of earnings. Subtracting the $282.59 half-SE-tax deduction gives earned income of $30,517.41 and a refundable CTC of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It understated the EITC at $8,162 instead of the full $8,231 2026 maximum, since AGI is below the phase-out threshold. It also computed the ACTC on gross $30,800 instead of $30,517.41 after the half-SE-tax deduction." -us,scenario_008,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of $4,203 was correct, but it overstated the 2026 three-child EITC maximum as $8,342 instead of $8,231." -us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,other,False,"It treated refundable credits as inapplicable because tax liability was already zero. Refundable credits are paid out regardless of liability, so the $8,231 EITC and the $4,202.61 refundable CTC are both payable in full." -us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules: a $3,000 ACTC earnings threshold and a $1,000-per-child credit. The permanent 2026 law uses a $2,500 threshold and a $1,700 refundable cap, which gives an ACTC of $4,202.61. It also overstated the EITC maximum as $8,247 instead of $8,231." -us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the expired $3,000 ACTC earnings threshold instead of the permanent $2,500 threshold. That gave $4,124 instead of $4,202.61. It also rounded the EITC to $8,250 instead of $8,231." -us,scenario_008,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its EITC of $8,231 was correct, but it assumed the TCJA had sunset and used a $3,000 ACTC threshold on $30,494 of earned income. Permanent law keeps the $2,500 threshold, and earned income is $30,517.41, so the ACTC is $4,202.61." -us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It phased the EITC down to $7,901, but AGI of about $30,632 is below the 2026 MFJ phase-out threshold, so the full $8,231 applies. Its ACTC used $30,494 of earned income instead of $30,517.41." -us,scenario_008,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 EITC maximum as $8,271 instead of $8,231. It also computed the ACTC on gross $30,800 of earnings instead of $30,517.41 after the half-SE-tax deduction." -us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer or explanation for federal_refundable_credits, so no value could be scored." -us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but its refundable CTC of $4,245 uses gross $30,800 of earnings. Earned income must subtract half of SE tax ($282.59), which gives 15% × $28,017.41 = $4,202.61." -us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a three-child EITC maximum of about $7,152 (closer to the two-child maximum) and phased it out from a $28,120 threshold. The 2026 three-child maximum is $8,231, and it is unreduced at this AGI. Its ACTC also used gross $30,800 instead of $30,517.41." -us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It overstated the 2026 EITC maximum as $8,262 instead of $8,231. It also computed the ACTC on gross $30,800 instead of $30,517.41 after the half-SE-tax deduction." -us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It added a $1,000 refundable AOTC even though qualified education expenses are $0, so the AOTC is $0. It also used a 2025-level $8,046 EITC maximum phased out from a $26,210 threshold, which gave $7,144. The full 2026 maximum of $8,231 applies because AGI is below the MFJ threshold." -us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It wrongly limited the ACTC to $1,600 of 'tax liability room'. The ACTC is 15% of earned income above $2,500 ($4,202.61) and does not depend on liability. It also understated the EITC at $6,801 instead of $8,231, and its total of $6,586 does not equal its own components." +us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model assumed the joint-filer EITC phase-out starts near $30,050 and cut the EITC to $8,173. AGI of $30,632 is below the 2026 threshold, so the full $8,231 applies. It also built ACTC on 0.9235 × SE income ($4,199) instead of SE income minus half the SE tax ($4,202.61), then reported $12,130, which does not even match its own $12,372 sum." +us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model put the EITC at only $1,395, but a 3+-child joint filer with about $30.5k of earned income gets the full $8,231. It also computed ACTC as $4,245, then used an unexplained $1,305 in the total instead of the correct $4,202.61." +us,scenario_008,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model admitted the EITC is positive but still reported zero refundable credits. This working joint-filer household with six children gets the full $8,231 EITC plus $4,202.61 of ACTC (15% of $28,017.41)." +us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's ACTC of $4,202 used the correct earned income ($30,517) and was right, and its EITC was close (it used an $8,267 maximum instead of $8,231). It then inflated its own $12,469 subtotal to $13,167 with an unexplained 'adjustment' instead of reporting $8,231 + $4,202.61." +us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model added a $1,000 refundable AOTC, but qualified education expenses are zero, so the AOTC is $0. It also cut ACTC to $388 on a mistaken assumption that nonrefundable use absorbs it. With zero tax liability, the full ACTC of 15% × ($30,517.41 − $2,500) = $4,202.61 is refundable." +us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model cut the EITC to about $7,700, but AGI is below the 2026 joint-filer phase-out start, so the full $8,231 applies. It computed ACTC on gross $30,800 instead of earned income net of the half-SE-tax deduction ($30,517.41), and it added a $355 AOTC even though qualified expenses are zero." +us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used 2025 parameters with a $26,511 phase-out start and an $8,046 maximum, giving a reduced EITC of $7,178. The 2026 3+-child maximum is $8,231, and the joint-filer phase-out begins above AGI of $30,632, so no reduction applies. Its ACTC used 0.9235 × SE income ($4,199) instead of SE income minus half the SE tax ($4,202.61)." +us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model applied an EITC phase-out that does not apply (giving $7,856 instead of $8,231). It added a $1,000 AOTC even though education expenses are zero. It also first treated ACTC as the $8,500 per-child cap even though the binding limit is 15% × ($30,517.41 − $2,500) = $4,202.61, then picked an unsupported $12,900." +us,scenario_008,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model computed ACTC on gross earned income of $30,800 ($4,245) without subtracting the $282.59 half-SE-tax deduction, which gives $4,202.61. It also reduced the EITC to $8,021 through a phase-out that does not apply, instead of the full $8,231." +us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the pre-TCJA $3,000 ACTC earned-income threshold instead of the $2,500 threshold in force for 2026, and it applied that threshold to gross $30,800. It also understated the EITC at $7,787 instead of the full $8,231 maximum." +us,scenario_008,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model's ACTC of $4,124 equals 15% × ($30,494 − $3,000), so it used the expired $3,000 threshold instead of $2,500 and the wrong earned income ($30,494 instead of $30,517.41). Its EITC of $8,190 also falls short of the full $8,231 maximum." +us,scenario_008,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model counted SE earnings as 0.9235 × $4,000 ($30,494 total) instead of $4,000 minus the $282.59 half-SE-tax deduction ($30,517.41), which put ACTC at $4,199.10 instead of $4,202.61. It also overstated the 3+-child EITC maximum as $8,247 instead of $8,231." +us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model cut the EITC to $7,691 even though AGI of $30,632 is below the 2026 joint-filer phase-out start, so the full $8,231 applies. Its ACTC used $30,494 of earned income (0.9235 factor) instead of $30,517.41, giving $4,199 instead of $4,202.61." +us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model claimed the maximum EITC plus the CTC but reported $11,400 with no computation. The correct figures are $8,231 of EITC plus ACTC of 15% × ($30,517.41 − $2,500) = $4,202.61, so the model understated the total by about $1,034." +us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's ACTC of $4,124 corresponds to the expired $3,000 threshold applied to $30,494, instead of 15% × ($30,517.41 − $2,500) = $4,202.61. It also rounded the EITC down to about $8,200 instead of $8,231." +us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model reduced the EITC to $7,128 with a phase-out that does not apply at AGI of $30,632; the full $8,231 maximum applies. Its ACTC of $4,128 reflects a $3,000 threshold instead of $2,500 on $30,517.41 of earned income ($4,202.61)." +us,scenario_008,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave $9,750 with no breakdown, which is $2,684 below the correct $8,231 EITC plus $4,202.61 ACTC. That shortfall means it undercounted either the full 3+-child EITC maximum or the 15%-of-earnings-over-$2,500 ACTC." +us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's ACTC of $4,203 was correct, but it used the 2025 3+-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231." +us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model gave only a $11,940 total, $494 short of $8,231 EITC plus $4,202.61 ACTC. That shortfall is consistent with reducing the EITC below its full maximum even though AGI of $30,632 is below the 2026 joint-filer phase-out start." +us,scenario_008,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's ACTC of $4,128 reflects a $3,000 threshold, or understated earned income, instead of 15% × ($30,517.41 − $2,500) = $4,202.61. It also used $8,233 instead of $8,231 for the EITC." +us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No answer or explanation was returned for federal_refundable_credits, so nothing could be scored." +us,scenario_008,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model overstated the 2026 3+-child EITC maximum as $8,360 instead of $8,231. Its ACTC used 0.9235 × SE income ($30,494 of earned income), giving $4,199 instead of $4,202.61 computed on SE income minus the half-SE-tax deduction." +us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model claimed the facts did not establish qualifying children. Six resident children ages 1–17 are listed, so the household qualifies for the full $8,231 EITC and $4,202.61 of ACTC." +us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model reported zero despite about $30.5k of earned income and six qualifying children. These facts produce an $8,231 EITC plus $4,202.61 of refundable ACTC." +us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model's EITC of $8,231 was correct, but it computed ACTC on gross earned income of $30,800 ($4,245). Earned income must exclude the $282.59 half-SE-tax deduction, giving $30,517.41 and an ACTC of $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum of $8,046 instead of the 2026 maximum of $8,231. It also computed ACTC on gross $30,800 ($4,245) instead of earned income net of the half-SE-tax deduction ($30,517.41 → $4,202.61)." +us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's EITC of $8,231 was correct, but its ACTC of $4,245 is 15% of gross $30,800 minus $2,500. Earned income has to subtract the $282.59 half-SE-tax deduction, which gives $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $12,476 equals $8,231 of EITC plus a $4,245 ACTC computed on gross $30,800 of earnings. That skips subtracting the $282.59 half-SE-tax deduction from SE income, which gives an ACTC of $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model computed ACTC on gross $30,800 ($4,245) instead of earned income net of the half-SE-tax deduction ($30,517.41 → $4,202.61). It also put the EITC at $8,162 instead of the full $8,231 maximum." +us,scenario_008,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model's ACTC of $4,203 was correct, but it overstated the 2026 3+-child EITC maximum as $8,342 instead of $8,231." +us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model treated refundable credits as inapplicable because net tax was already zero. Refundable credits are paid out regardless of tax liability, so the household receives $8,231 of EITC and $4,202.61 of ACTC." +us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the pre-TCJA ACTC rules, a $3,000 threshold and $1,000 per child, but the 2026 rules use a $2,500 threshold and a $1,700 refundable cap. That gave $4,124 instead of $4,202.61. It also used $8,247 instead of $8,231 for the EITC." +us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied the expired $3,000 ACTC threshold instead of $2,500, giving $4,124 instead of 15% × ($30,517.41 − $2,500) = $4,202.61. It also rounded the EITC to $8,250 instead of $8,231." +us,scenario_008,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model's EITC of $8,231 was correct, but it assumed the TCJA sunset and used a $3,000 ACTC threshold. The 2026 threshold is $2,500. It also used 0.9235 × SE income ($30,494) instead of SE income minus half the SE tax ($30,517.41), yielding $4,124.10 instead of $4,202.61." +us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied an EITC phase-out at AGI of $30,632 and got $7,901, but that AGI is below the 2026 joint-filer phase-out start, so the full $8,231 applies. Its ACTC also used $30,494 of earned income ($4,199) instead of $30,517.41 ($4,202.61)." +us,scenario_008,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model computed ACTC on gross $30,800 ($4,245) without subtracting the half-SE-tax deduction, which gives $4,202.61. It also overstated the 2026 EITC maximum as $8,271 instead of $8,231." +us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for federal_refundable_credits, so nothing could be scored." +us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model's EITC of $8,231 was correct, but it computed ACTC as 15% of gross $30,800 minus $2,500 ($4,245). It ignored the $282.59 half-SE-tax deduction that reduces earned income to $30,517.41 and ACTC to $4,202.61." +us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a $7,152 maximum and a $28,120 phase-out start, which produced a $6,588 EITC. The 2026 3+-child maximum is $8,231, and no phase-out applies at AGI of $30,632. Its ACTC used gross $30,800 ($4,245) instead of $30,517.41 ($4,202.61)." +us,scenario_008,federal_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum of $8,046 instead of the 2026 maximum of $8,231. Its ACTC used $30,494 of earned income ($4,199) instead of $30,517.41 net of the half-SE-tax deduction ($4,202.61)." +us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model computed ACTC on gross $30,800 ($4,245) instead of earned income reduced by the $282.59 half-SE-tax deduction ($4,202.61). It also overstated the EITC maximum as $8,262 instead of $8,231." +us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model added a $1,000 refundable AOTC even though qualified education expenses are zero. It also applied an EITC phase-out from a $26,210 threshold, getting $7,144 instead of the full $8,231. Its ACTC used $30,494 of earned income ($4,199) instead of $30,517.41 ($4,202.61)." +us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model capped ACTC at $1,600 based on 'remaining tax liability room', but ACTC is limited only by 15% of earned income over $2,500 ($4,202.61) and the per-child cap. It understated the EITC at $6,801 instead of $8,231, and its total of $6,586 does not equal its own components." us,scenario_008,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required a separately provided school-meals eligibility indicator and treated housing assistance as the only relevant benefit fact. It failed to compute the income-based free tier at 55% of the federal poverty guideline and also missed categorical eligibility through the household's computed SNAP eligibility. us,scenario_008,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated the head as ineligible because they are a non-disabled adult. It ignored NJ's ACA expansion pathway, which covers any adult under 65 with MAGI at or below 138% FPL. The head's 8-person household MAGI is 0.55× FPL, so they qualify in the ADULT category." us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income exceeds NJ guidelines, but it never compared household MAGI (about $30.8k) with 138% FPL for an 8-person household (about $77k). The household is at 0.55× FPL, far below the expansion limit, so the head is eligible." @@ -589,102 +598,104 @@ us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibilit us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly found the head in the expansion category with income below 138% FPL. It then applied a made-up rule that receiving housing assistance disqualifies adults from Medicaid. No such exclusion exists, so the head's ACA adult expansion eligibility at 0.55× FPL stands." us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly transferred the infant's categorical eligibility to the Head through household membership. WIC requires the applicant personally to be a pregnant, postpartum, or breastfeeding woman, infant, or child under age 5; the 42-year-old Head meets none of those categories and is therefore ineligible despite household income below the limit." us,scenario_008,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used 2024 NJ rates: TDI at 0% instead of 0.19% ($50.92) and FLI at 0.09% ($24.12) instead of 0.23% ($61.64). It also summed its own NJ items ($102.51 + $24.12 + $11.39 = $138.02) as $128.65, so its NJ total came out $97.81 short of $226.46." -us,scenario_008,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal FICA ($2,050.20) and left out the $226.46 of mandatory NJ employee contributions: TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425% on $26,800. The question explicitly lists these as part of payroll_tax." -us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Got FICA right ($2,050.20), then dropped its own NJ UI estimate and replaced it with a made-up $2.12 'minimum assessment'. The correct NJ employee contributions total $226.46: TDI $50.92, FLI $61.64, UI $102.51 and workforce $11.39." -us,scenario_008,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Stopped at federal FICA of $2,050.20 and never added NJ's mandatory employee TDI (0.19%), FLI (0.23%), UI (0.3825%) or workforce (0.0425%) contributions. Those total $226.46 on $26,800." -us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Recognized that NJ has employee UI, SDI and FLI contributions, but left all of them out because it was unsure of the rates. It wrongly took the 2026 TDI rate to be 0%, when the correct 2026 rates are TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%, which add $226.46." -us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Assumed NJ's mandatory employee UI, SDI and FLI contributions were 'not modeled' and set them to $0. They are part of payroll_tax and come to $226.46 on $26,800 (TDI $50.92, FLI $61.64, UI $102.51, workforce $11.39)." -us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Explicitly excluded NJ employee UI, TDI and FLI from the measure, even though the question defines payroll_tax to include mandatory employee state payroll taxes. That left out $226.46, and it also rounded FICA down to $2,050." -us,scenario_008,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"Applied only the 7.65% federal employee FICA rate. It left out NJ's mandatory employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%) contributions, which total $226.46." -us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Wrongly concluded that NJ TDI is employer-funded and that NJ employee contributions fall outside 'mandatory employee state payroll taxes', so it used FICA alone. NJ withholds employee TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%, which add $226.46." -us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Estimated about $201 of NJ UI/DI/FLI contributions, then dropped them from the total, citing wage-cap uncertainty. $26,800 is below every NJ wage base, so the full $226.46 applies: TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%." -us,scenario_008,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed NJ employee payroll taxes are 'not modeled' and counted only FICA. That left out $226.46 of mandatory NJ employee TDI, FLI, UI and workforce contributions on $26,800." -us,scenario_008,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Counted only federal Social Security and Medicare, then rounded to $2,050. It never added the $226.46 of mandatory NJ employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%) contributions." -us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Included only NJ TDI at 0.10% ($26.80) and FLI at 0.09% ($24.12). The 2026 rates are 0.19% ($50.92) and 0.23% ($61.64), and it left out the employee UI (0.3825%, $102.51) and workforce (0.0425%, $11.39) contributions entirely." -us,scenario_008,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Put NJ employee contributions at $154.10, a combined 0.575% of wages. The correct combined rate is 0.845% ($226.46): TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%. That leaves the NJ component $72.36 short." -us,scenario_008,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Computed only employee FICA of $2,050.20. It left out the mandatory NJ employee TDI, FLI, UI and workforce contributions, which add $226.46 on $26,800." -us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Applied a combined 0.515% NJ rate ($138.02), which is UI/workforce 0.425% plus the 2024 FLI rate of 0.09%. It left out the 0.19% employee TDI and used 0.09% instead of the 2026 FLI rate of 0.23%, so NJ came out $88.44 short." -us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Applied only the 7.65% federal FICA rate and rounded to $2,050. It left out the $226.46 of mandatory NJ employee TDI, FLI, UI and workforce contributions." -us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Estimated NJ contributions at about $138, consistent with UI/workforce 0.425% plus 2024 FLI 0.09% and no TDI. The 2026 rates add TDI 0.19% and FLI 0.23%, for $226.46 in total." -us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"Counted only NJ UI and FLI, at $123.80 (about 0.46%). It left out the 0.19% employee TDI and understated FLI and workforce relative to the 2026 total of $226.46 (TDI $50.92, FLI $61.64, UI $102.51, workforce $11.39)." -us,scenario_008,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Applied only 7.65% FICA to the spouse's wages. It never added NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46." -us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"Counted NJ SUI/workforce (0.425%) and FLI at the 2024 rate of 0.09%, for $138.02. It left out the 0.19% employee TDI ($50.92) and understated FLI, whose 2026 rate is 0.23% ($61.64)." -us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Included only NJ FLI, SUI and workforce, for $139.32. It left out the 0.19% employee TDI and used an FLI rate far below the 2026 rate of 0.23%, so it missed $87.14 of the $226.46 NJ total." -us,scenario_008,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Stopped at Social Security and Medicare ($2,050.20). It left out NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46." -us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value. The correct figure is $2,050.20 of FICA plus $226.46 of NJ employee contributions, or $2,276.66." -us,scenario_008,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Treated the NJ employee state payroll tax as 'not modeled' and reported FICA only. It left out the $226.46 of NJ employee TDI, FLI, UI and workforce contributions." -us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its own components sum to $2,050.20 ($1,661.60 + $388.60), but it submitted $4,097, roughly double, which is an arithmetic failure. It also stated that no mandatory state employee payroll tax applies, leaving out NJ's $226.46." -us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Submitted $1,525, which is below even the 7.65% federal FICA on $26,800 ($2,050.20), so the federal calculation itself is wrong. It also left out the $226.46 of NJ employee TDI, FLI, UI and workforce contributions." -us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"Applied a combined 0.9425% NJ employee rate ($252.59). The 2026 rates sum to 0.845%: UI 0.3825%, workforce 0.0425%, TDI 0.19% and FLI 0.23%. That overstates the NJ contributions by $26.13." -us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"Its NJ figure of $253.26 is 0.945% of wages, 0.10 percentage points above the 2026 combined rate of 0.845%. That gap equals using the 2025 FLI rate (0.33%) instead of the 2026 rate (0.23%), and overstates FLI by $26.80." -us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Applied the 2025 NJ rates of TDI 0.23% and FLI 0.33% (0.985% combined, $263.98). The 2026 rates are 0.19% and 0.23%, for a combined 0.845% ($226.46), so NJ is overstated by $37.52." -us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Estimated NJ employee contributions at $202.34, a combined 0.755% of wages. The 2026 combined rate is 0.845%: UI 0.3825%, workforce 0.0425%, TDI 0.19% and FLI 0.23%, totaling $226.46. That leaves NJ $24.12 short." -us,scenario_008,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Used the 2025 NJ combined employee rate of 0.985% (TDI 0.23%, FLI 0.33%, UI/workforce 0.425%), giving $263.98. The 2026 rates are TDI 0.19% and FLI 0.23%, which give $226.46." -us,scenario_008,payroll_tax,gpt-6.1-sol,llm_error,state_local_rule,False,"Its NJ total of $215.07 is exactly $11.39 below the correct $226.46. That is the amount of the 0.0425% workforce development/supplemental workforce fund contribution, so that component was omitted from the four NJ employee contributions." -us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Applied only 7.65% FICA (rounded to $2,050). It never added NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46." -us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"Counted only NJ TDI plus FLI, at a combined 0.18% ($48.24). The 2026 TDI and FLI rates are 0.19% and 0.23%, which come to $112.56, and it left out the employee UI (0.3825%, $102.51) and workforce (0.0425%, $11.39) contributions." -us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"Included UI at 0.3825% but estimated TDI and FLI at only about $75 and left out the 0.0425% workforce contribution. Its NJ total of $178 falls short of the $226.46 produced by TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%." -us,scenario_008,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"Used the 2025 NJ rates of TDI 0.23% and FLI 0.33% (0.985% combined, $263.98). The 2026 rates are TDI 0.19% and FLI 0.23%, a combined 0.845% ($226.46)." -us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"Explicitly assumed no state payroll taxes apply. NJ imposes mandatory employee TDI, FLI, UI and workforce contributions of $226.46 on the spouse's $26,800." -us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"Counted only NJ UI at 0.3825% and FLI at the 2024 rate of 0.09%. It left out the 0.0425% workforce contribution and the 0.19% employee TDI, and understated FLI, whose 2026 rate is 0.23%." -us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value. The correct figure is $2,050.20 of FICA plus $226.46 of NJ employee contributions, or $2,276.66." -us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Estimated NJ UI/workforce, TDI and FLI at $257.95 (about 0.9625%), above the 2026 combined rate of 0.845% ($226.46). It overstated the TDI and FLI rates, which are 0.19% and 0.23% for 2026." -us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Computed only federal Social Security and Medicare (rounded to $2,050). It left out NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46." -us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Assumed NJ employee payroll tax is not modeled and reported FICA only. It left out the $226.46 of NJ employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%) contributions." -us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Added only NJ UI at 0.3825% ($102.51) and assumed the 2024 TDI and FLI employee rates of 0%. The 2026 rates are TDI 0.19% ($50.92) and FLI 0.23% ($61.64), and it also left out the 0.0425% workforce contribution ($11.39)." -us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Applied only 7.65% FICA to the spouse's wages. It left out NJ's mandatory employee TDI, FLI, UI and workforce contributions of $226.46." +us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It correctly included NJ UI (0.3825%, $102.51) and the workforce fund (0.0425%, $11.39). However, it set the TDI employee rate to 0% instead of 0.19% ($50.92) and used 0.09% for FLI instead of 0.23% ($61.64). It also added its own components wrong: $102.51 + $24.12 + $11.39 is $138.02, not the $128.65 it reported." +us,scenario_008,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA ($2,050.20) and left out NJ's mandatory employee contributions on the $26,800 of wages. Those are UI $102.51, workforce fund $11.39, TDI $50.92 and FLI $61.64, totaling $226.46." +us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It added an invented $2.12 NJ 'minimum assessment' to FICA. It should have applied NJ's employee rates to $26,800: UI 0.3825%, workforce fund 0.0425%, TDI 0.19% and FLI 0.23%, which come to $226.46." +us,scenario_008,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It stopped at federal FICA of $2,050.20. It left out the $226.46 of mandatory NJ employee UI, workforce-fund, TDI and FLI contributions that the output definition explicitly includes." +us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It identified NJ UI, SDI and FLI but dropped them over rate uncertainty, and it wrongly assumed a 0% 2026 TDI rate. The 2026 employee rates are TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce fund 0.0425%, which add $226.46 to FICA." +us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It wrongly treated NJ UI, SDI and FLI employee contributions as not modeled and set them to $0. These mandatory employee state payroll taxes total $226.46 on $26,800 of wages." +us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It explicitly excluded NJ employee UI, TDI and FLI, even though the output definition counts mandatory employee state payroll taxes, so it missed $226.46. It also rounded FICA down to $2,050." +us,scenario_008,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It applied only the 7.65% federal FICA rate. It left out NJ's 0.845% combined employee contribution (UI, workforce fund, TDI and FLI), which is $226.46." +us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly claimed that NJ TDI is employer-funded with a 0% employee rate, and it treated FLI as not a payroll tax. NJ withholds employee TDI at 0.19%, FLI at 0.23%, UI at 0.3825% and the workforce fund at 0.0425%, which total $226.46." +us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It estimated about $201 of NJ employee UI/DI/FLI contributions but then set them to $0 citing cap uncertainty. The wages are below every NJ wage base, so the full $226.46 applies." +us,scenario_008,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It wrongly asserted that NJ employee payroll taxes are not modeled. That omitted the $226.46 of UI, workforce-fund, TDI and FLI contributions on the spouse's wages." +us,scenario_008,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out NJ's mandatory employee contributions of $226.46. It also rounded $2,050.20 down to $2,050." +us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It included only NJ TDI and FLI, at understated rates of 0.10% and 0.09% instead of 0.19% and 0.23%. It also left out NJ employee UI (0.3825%, $102.51) and the workforce fund (0.0425%, $11.39)." +us,scenario_008,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It put NJ employee payroll taxes at $154.10, about 0.575% of wages. The correct combined rate is 0.845%: UI 0.3825%, workforce fund 0.0425%, TDI 0.19% and FLI 0.23%, totaling $226.46." +us,scenario_008,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed only federal FICA and left out the $226.46 of mandatory NJ employee UI, workforce-fund, TDI and FLI contributions." +us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Its 0.515% NJ rate covers UI plus the workforce fund (0.425%) plus FLI at 0.09%. That omits the 0.19% TDI employee contribution and understates FLI, which is 0.23%. It also rounded the total to $2,188." +us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only the 7.65% FICA rate (rounded to $2,050). It left out the $226.46 of NJ employee UI, workforce-fund, TDI and FLI contributions." +us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It estimated NJ contributions at about $138, which matches UI plus the workforce fund plus FLI at 0.09% with no TDI. The correct figure adds TDI at 0.19% and uses FLI at 0.23%, for $226.46." +us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It counted only NJ UI/FLI at $123.80. That left out the 0.19% TDI contribution ($50.92) and the 0.0425% workforce fund ($11.39), and it understated FLI, which is 0.23% ($61.64)." +us,scenario_008,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,It applied 7.65% FICA only and left out NJ's mandatory employee contributions of $226.46. +us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $138.02 of NJ SUI/FLI equals UI $102.51 plus the workforce fund $11.39 plus FLI at 0.09% ($24.12). It omitted TDI at 0.19% ($50.92) and understated FLI, which is 0.23% ($61.64)." +us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"It included only NJ FLI, SUI and the workforce fund, about $139.32, and left out the 0.19% TDI employee contribution. It also understated FLI, which is 0.23%, so it fell short of the $226.46 NJ total." +us,scenario_008,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out the $226.46 of mandatory NJ employee UI, workforce-fund, TDI and FLI contributions." +us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there is no answer to score." +us,scenario_008,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It explicitly treated the NJ employee state payroll tax as not modeled. That omitted UI $102.51, workforce fund $11.39, TDI $50.92 and FLI $61.64." +us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly computed $1,661.60 and $388.60 in FICA, then reported an unrelated $4,097, roughly double its own FICA sum. It also left out NJ's $226.46 of employee contributions." +us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $1,525 is about 5.7% of wages, below even the 7.65% employee FICA rate of $2,050.20. It also left out the $226.46 of NJ employee UI, workforce-fund, TDI and FLI contributions." +us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It applied a 0.9425% combined NJ employee rate, which overstates the TDI and FLI rates. The correct combined rate is 0.845% (UI 0.3825%, workforce fund 0.0425%, TDI 0.19%, FLI 0.23%), which gives $226.46 rather than $252.59." +us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $253.26 of NJ contributions implies about 0.945% of wages. That overstates the 2026 TDI (0.19%) and FLI (0.23%) employee rates, which together with UI and the workforce fund total 0.845%, or $226.46." +us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $263.98 is 0.985% of wages, which corresponds to TDI at 0.23% and FLI at 0.33%. The correct 2026 rates are 0.19% and 0.23%, so the NJ total is $226.46." +us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Its NJ estimate of $202.34 (about 0.755%) understates the combined employee rate. The correct rate is 0.845%: UI 0.3825%, workforce fund 0.0425%, TDI 0.19% and FLI 0.23%, totaling $226.46." +us,scenario_008,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Its $263.98 of NJ contributions (0.985%) uses outdated TDI 0.23% and FLI 0.33% rates instead of the 2026 rates of 0.19% and 0.23%. That overstates the NJ total, which is $226.46." +us,scenario_008,payroll_tax,gpt-6.1-sol,llm_error,state_local_rule,False,Its $215.07 NJ total falls short by exactly the $11.39 workforce-fund contribution. That is consistent with treating the 0.3825% UI rate as already including the separate 0.0425% workforce fund. +us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only the 7.65% FICA rate and rounded it to $2,050. It left out NJ's mandatory employee UI, workforce-fund, TDI and FLI contributions of $226.46." +us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It included only NJ TDI plus FLI, at a combined 0.18% instead of 0.42% (0.19% + 0.23%). It also omitted NJ employee UI (0.3825%) and the workforce fund (0.0425%)." +us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It estimated NJ UI plus TDI/FLI at about $178 and never applied the 0.0425% workforce fund. It also underestimated TDI (0.19%) and FLI (0.23%), so it fell short of the $226.46 NJ total." +us,scenario_008,payroll_tax,grok-4.7,llm_error,state_local_rule,False,It used TDI 0.23% and FLI 0.33% instead of the 2026 employee rates of 0.19% and 0.23%. That overstated NJ contributions at $263.98 instead of $226.46. +us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It explicitly said there were no state payroll taxes. NJ requires employee UI, workforce-fund, TDI and FLI contributions of $226.46 on these wages, and the model also rounded FICA to $2,050." +us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"It counted only NJ UI and FLI, with FLI at 0.09%, for about $127. It omitted TDI at 0.19% ($50.92) and the workforce fund at 0.0425% ($11.39), and understated FLI, which is 0.23% ($61.64)." +us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there is no answer to score." +us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,Its $257.95 of NJ contributions (about 0.9625%) overstates the TDI and FLI employee rates. The correct 0.845% combined rate gives $226.46. +us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It computed only Social Security and Medicare, rounded to $2,050. It never added NJ's mandatory employee UI, workforce-fund, TDI and FLI contributions of $226.46." +us,scenario_008,payroll_tax,muse-wonder,llm_error,state_local_rule,False,"It estimated about $241 of NJ unemployment, disability and family-leave contributions, which overstates the 0.845% combined employee rate. The correct NJ amount is $226.46, and the model also rounded FICA." +us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It wrongly treated the NJ employee payroll tax as not modeled. That omitted $226.46 of UI, workforce-fund, TDI and FLI contributions on the spouse's $26,800." +us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It included only NJ UI at 0.3825% and set TDI and FLI to 0% based on outdated rates. In 2026 they are 0.19% ($50.92) and 0.23% ($61.64), and the model also omitted the 0.0425% workforce fund ($11.39)." +us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied 7.65% FICA only and left out the $226.46 of mandatory NJ employee UI, workforce-fund, TDI and FLI contributions." us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling and housing assistance as sufficient for reduced-price support, without applying the mutually exclusive free-meals tier. At 55% of the federal poverty guideline and with categorical eligibility through SNAP/TANF, the household qualifies for free meals, which makes reduced-price eligibility false." us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract." us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $306, exactly 7.65% of $4,000, despite stating that it applied 15.3%. It effectively used only half the combined Social Security and Medicare self-employment tax rate and also failed to apply the 92.35% statutory earnings factor, instead of calculating $4,000 × 0.9235 × 0.153 = $565.18." us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract." us,scenario_008,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated gross self-employment tax of $565.18, then incorrectly subtracted a supposed deduction from that tax liability. The deductible half of self-employment tax reduces adjusted gross income for income-tax purposes; it is not a credit against self-employment tax, so the liability remains $565.18." -us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used outdated parameters: a $258 standard deduction instead of the FY2026 $299 for 6+ people, and a $1,756 maximum allotment instead of $1,789. It computed $1,214/month and then replaced that with an unsupported $687/month. Correct inputs give $1,763.92 net income (rounded to $1,764), a $530 contribution and $1,259/month." -us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $302 standard deduction instead of the FY2026 $299 for 6+ people, so net income came out at $1,760.92 instead of $1,763.92. It also did not round net income to $1,764 or round the 30% contribution up to $530, which produced $1,260.72/month instead of $1,259." -us,scenario_008,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the 20% earned income deduction ($513.33/month) and used a $227 standard deduction instead of $299, which overstated net income at $2,349. It also used a $1,164 maximum allotment, far below the FY2026 $1,789 for eight people." -us,scenario_008,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It denied SNAP because of the $69,500 in bank assets. The household is categorically eligible through New Jersey's TANF non-cash (BBCE) pathway, which removes the asset test, and its gross income is only 57% of the poverty guideline." -us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted a monthly standard deduction (about $280) once from annual income instead of $299 every month. That overstated monthly net income at $2,040 instead of $1,764. It also used the outdated $1,756 maximum allotment and then gave an arbitrary $13,620 that differs from its own $13,728." -us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It subtracted the $258 monthly standard deduction once from annual income and used $258 instead of the FY2026 $299, so monthly net income came out at $2,041 instead of $1,764. It also used the outdated $1,756 maximum allotment and then cut its own $13,728 result to an unsupported $11,352." -us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"It used a $1,802 maximum allotment instead of $1,789. Its numbers also contradict each other: $1,802 minus 30% of $1,700 is $1,292, not $1,040. The correct net income of $1,764 gives a $530 contribution and $1,259/month." -us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"Its inputs were right, but it did not round net income to the whole dollar ($1,764) or round the 30% contribution up to the next whole dollar ($530). That gave $1,259.82/month instead of $1,259." -us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $1,536 maximum allotment instead of the FY2026 $1,789 for eight people. It also used a $258 standard deduction instead of $299 for 6+ people, which produced $995/month instead of $1,259." -us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up an excess shelter deduction with a utility allowance even though no utility costs were listed, and it applied a disability deduction. That pushed net income down to about $8,800/year. The $579 rent is below half of the $1,764 net income, so there is no shelter deduction. It also used a $204 standard deduction and a $1,691 maximum allotment, and its own arithmetic gives about $17,652, not $11,500." -us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"It correctly found $1,764 net income and the $1,789 maximum allotment, but it did not round the 30% contribution ($529.20) up to $530. As a result it rounded the benefit to $1,260/month instead of $1,259." -us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated net income at about $1,830 instead of $1,764 even while claiming a shelter deduction that does not apply. It also used about $1,780 as the maximum allotment instead of $1,789." -us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used about $1,900 as the maximum allotment instead of the FY2026 $1,789 for eight people, and a $302 standard deduction instead of $299. It also skipped the rounding of the 30% contribution, which overstated the benefit at $1,371/month." -us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $1,756 instead of the FY2026 $1,789. Its net income of $1,783.92 implies a standard deduction of about $279 instead of $299, which produced $1,220.83/month instead of $1,259." -us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used an $1,800 maximum allotment instead of $1,789. Its net income of $1,752.92 implies a standard deduction of about $310 instead of $299. It also did not round net income or round the contribution up." -us,scenario_008,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"$15,300 works out to $1,275/month, or $16 above the correct $1,259 ($1,789 minus a $530 contribution on $1,764 net income). That matches an inflated maximum allotment or a shelter deduction it invoked. No shelter deduction applies because the $579 rent is below half of net income." -us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no calculation. $16,400 ($1,367/month) is about $108/month above the correct $1,259, which matches an inflated maximum allotment or understated net income rather than $1,789 minus 30% of $1,764 rounded up to $530." -us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,307/month is $48 above the correct $1,259. That matches a maximum allotment higher than the FY2026 $1,789, or net income understated by about $160 below the correct $1,764 (which carries a $530 contribution)." -us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added New Jersey's standard utility allowance to rent to create an excess shelter deduction, even though no utility expenses were listed. Without it, the $579 rent is below half of the $1,764 net income, so the shelter deduction is $0 and the benefit is $1,259/month." -us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no calculation. $14,500 ($1,208/month) is $51/month below the correct $1,259, which matches the outdated $1,756 FY2025 maximum allotment rather than the FY2026 $1,789 minus a $530 contribution." -us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $1,756 instead of the FY2026 $1,789, and a $287 standard deduction instead of $299. That gave $1,775.92 net income and $1,223/month instead of $1,259." -us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no calculation. $16,848 ($1,404/month) is $145/month above the correct $1,259, which matches a much higher maximum allotment or deductions not in the facts, not $1,789 minus 30% of $1,764 rounded up to $530." -us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It applied a shelter deduction that does not exist here: the $579 rent is below half of the $1,764 net income. The resulting $1,317/month is above the correct $1,259 ($1,789 minus $530)." -us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted a $198 standard deduction once from annual income instead of $299 every month, which overstated net income at $24,557/year. It also used a $1,748 maximum allotment instead of $1,789." -us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $210 standard deduction instead of the FY2026 $299 for households of 6+, so net income came out at about $1,853 instead of $1,764. It also used a $1,797 maximum allotment instead of $1,789." -us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It gave $0 with no supporting test. The household's gross income is 57% of the poverty guideline, it passes both income tests, and it is categorically eligible through New Jersey BBCE. Housing assistance does not disqualify it, so it receives $1,259/month." -us,scenario_008,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It refused to compute and gave $0, even though the facts provided every input needed. With $2,576.25 gross income, $299 standard and $513.33 earned income deductions, and categorical eligibility through BBCE, the benefit is $1,259/month." -us,scenario_008,snap,gpt-5.5,llm_error,other,False,"It used the right deductions and maximum allotment, but it did not round the 30% contribution ($529.20) up to $530. As a result it rounded the benefit to $1,260/month instead of $1,259." -us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $1,139/month is $120 below the correct $1,259 and implies net income of about $2,167 against the $1,789 maximum allotment. That means it understated the standard and earned income deductions, which total $812.33 and give $1,764 net income. It also claimed a shelter deduction that does not apply." -us,scenario_008,snap,gpt-5.6-sol,llm_error,other,False,"Its $1,262/month implies a contribution of about $527 instead of $530. It did not round net income to $1,764 and round 30% of it up to $530, so the result is $3/month above $1,259." -us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $1,284.02/month implies a contribution of about $505, which corresponds to net income of about $1,683 instead of $1,764. That means it overstated deductions beyond the $299 standard and $513.33 earned income deductions, and it also skipped rounding the contribution up." -us,scenario_008,snap,gpt-6-astra,llm_error,other,False,"It computed on annual totals ($21,468 minus 30% of $21,167) without the monthly rounding steps. Net income needs to be rounded to $1,764 and the 30% contribution rounded up to $530 each month, which gives $1,259/month, not $1,259.825." -us,scenario_008,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $1,288/month is $29 above the correct $1,259. That matches a maximum allotment higher than the FY2026 $1,789, or net income understated below the correct $1,764 (which carries a $530 contribution)." -us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"It cut net income of $1,763.92 down to $1,763 instead of rounding to $1,764. It also used a 'roughly $529' contribution instead of rounding 30% up to $530, which produced $1,260/month instead of $1,259." -us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It wrongly treated housing assistance and income as making the household ineligible. Subsidized housing does not bar SNAP, gross income is 57% of the poverty guideline, and BBCE makes the household eligible for $1,259/month." -us,scenario_008,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It gave an excess medical deduction for the disabled spouse's over-the-counter costs, but $300/year is $25/month, below the $35 threshold, so no medical deduction applies. It also used about $1,822.50/month as the maximum allotment instead of $1,789." -us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"It used outdated parameters: about $1,759 as the maximum allotment instead of the FY2026 $1,789, and a standard deduction near the FY2025 $291 instead of $299. That gave $1,227/month instead of $1,259." -us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly carried over FY2025 parameters: a $1,756/month maximum allotment ($21,072/year) and a $291/month standard deduction ($3,492/year). FY2026 uses $1,789 and $299, with net income rounded to $1,764 and the contribution rounded up to $530." -us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the federal $4,250 resource limit for households with a disabled member and denied benefits over the $69,500 in bank assets. New Jersey BBCE (TANF non-cash categorical eligibility) waives the asset test entirely." -us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"It used an $1,806 maximum allotment instead of the FY2026 $1,789 for eight people. It also used a $529 contribution instead of rounding 30% of $1,764 up to $530." -us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SNAP value and no explanation, so there was no answer to score." -us,scenario_008,snap,kimi-k3,llm_error,other,False,"Its inputs were right ($1,763.92 net income, $1,789 maximum allotment), but it did not round net income to $1,764 or round the 30% contribution up to $530. That gave $1,259.825/month instead of $1,259." -us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It applied an asset limit (and an outdated $3,750 one) to deny SNAP over $69,500 in bank assets. New Jersey BBCE removes the asset test for this household." -us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,795 maximum allotment instead of $1,789 and a $297 standard deduction instead of $299. It also skipped rounding the 30% contribution up to $530, which gave $1,265/month instead of $1,259." -us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $202 standard deduction instead of the FY2026 $299 for 6+ people and a $1,838 maximum allotment instead of $1,789. It also left out the $115 interest and then cut the result by an unsupported 'housing assistance' adjustment." -us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It wrongly found that $2,567/month exceeds 130% of the poverty guideline for eight people. That limit is roughly $5,900/month, and the household's gross income is only 57% of the poverty guideline, so it passes the gross income test easily." +us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $1,756 and a $258 standard deduction instead of $1,789 and $299. It then threw out its own $1,214/month result for an unsupported $687/month figure. The correct path is $1,789 minus a $530 contribution, which gives $1,259/month." +us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $302 standard deduction instead of $299, which gives net income of $1,760.92 instead of $1,763.92. It also skipped the rounding steps: net income rounds to $1,764 and the 30% contribution rounds up to $530. That left it with a fractional $1,260.72/month instead of $1,259." +us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It left out the 20% earned income deduction ($513.33/month) and used a $227 standard deduction instead of $299. It also used an invented 8-person maximum allotment of $1,164 instead of $1,789, which roughly halved the benefit." +us,scenario_008,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It applied a SNAP resource limit to the $69,500 in bank assets. The household is categorically eligible through New Jersey's TANF non-cash BBCE, which removes the asset test. With gross income at 57% of the poverty guideline, the household receives $1,259/month." +us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted the roughly $280 monthly standard deduction once from annual income, which overstated monthly net income at about $2,040 instead of $1,764. It also used the FY2025 $1,756 maximum allotment instead of $1,789. It then reported $13,620, which doesn't match its own $13,728." +us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It subtracted the $258 monthly standard deduction once from $30,915 of annual income instead of $299 each month, which inflated net income to about $2,041/month. It used the stale $1,756 maximum allotment. It then cut its own $13,728 result to an unexplained $11,352." +us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"It used an $1,802 maximum allotment instead of $1,789. It also stated a $1,040/month benefit that its own inputs don't support: $1,802 minus 30% of $1,700 is about $1,292. The correct figure is $1,789 minus $530, or $1,259/month." +us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"Its inputs were right ($299 standard deduction, $513.33 earned income deduction, $1,789 maximum), but it skipped SNAP's rounding steps. Net income of $1,763.92 rounds to $1,764, and the $529.20 contribution rounds up to $530. That gives $1,259/month, not the fractional $1,259.82 it used." +us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used an 8-person maximum allotment of $1,536 instead of $1,789. It also used a $258 standard deduction instead of the $299 that applies to households of 6 or more. This raised net income to $1,805 and lowered the benefit to $995/month." +us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up an excess shelter deduction with a utility allowance, even though no utilities are listed and the $579 rent is below half of adjusted income. It also used a $204 standard deduction and a $1,691 maximum. Its net income of about $8,800/year is far below the correct $1,764/month." +us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"It had the right $1,764 net income and $1,789 maximum, but it rounded the 30% contribution ($529.20) down to $529. SNAP rounds the contribution up to $530, so the benefit is $1,259/month, not $1,260." +us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It reached net income of about $1,830 instead of $1,764, which means it understated the $299 standard deduction even while claiming a shelter deduction that doesn't apply. It also used a maximum allotment of about $1,780 instead of $1,789." +us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used an 8-person maximum allotment of about $1,900 instead of $1,789, and a $302 standard deduction instead of $299. That inflated the benefit to $1,371/month." +us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $1,756 instead of the FY2026 figure of $1,789. Its $1,783.92 net income implies a $279 standard deduction instead of $299." +us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a maximum allotment of $1,800 instead of $1,789. Its net income of $1,752.92 implies a $310 standard deduction instead of $299. It also skipped rounding net income to the dollar and rounding the contribution up." +us,scenario_008,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It said it applied a shelter deduction, which this household doesn't get: rent of $579/month is below half of adjusted income and no utilities are listed. Its $1,275/month is about $16 too high. The correct figure is $1,789 minus a $530 contribution, or $1,259." +us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no calculation, and its $16,400 (about $1,367/month) is above what the rules allow. With net income of $1,764 and a $530 contribution, the benefit is $1,789 − $530 = $1,259/month. Its number is consistent with an overstated maximum allotment or unsupported extra deductions." +us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,307/month is $48 too high. Correct parameters give $1,789 minus $530 ($1,764 net income × 30%, rounded up), or $1,259. The overstatement is consistent with a maximum allotment near $1,836 or overstated deductions." +us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied an excess shelter deduction using actual rent plus New Jersey's standard utility allowance. No utility costs are listed, and the $579 rent is below half of the $1,764 adjusted income, so the shelter deduction is $0. That pushed its benefit above $1,259/month." +us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no calculation, and its $14,500 (about $1,208/month) is $51/month short of $1,259. That shortfall matches using the stale $1,756 FY2025 maximum allotment instead of the FY2026 figure of $1,789." +us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $287 standard deduction instead of $299, and the FY2025 maximum allotment of $1,756 instead of $1,789. That gave $1,223/month instead of $1,259." +us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no calculation, and its $1,404/month is $145 above the correct $1,259 ($1,789 minus a $530 contribution on $1,764 net income). That is consistent with a maximum allotment near $1,934 or an unsupported deduction." +us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It said it applied a shelter deduction, but rent of $579/month is below half of adjusted income and no utilities are listed, so the shelter deduction is $0. Its $1,317/month overstates the correct $1,259." +us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted a $198 monthly standard deduction once from annual income instead of $299 every month, which inflated net income to $24,557/year. It also used a $1,748 maximum allotment instead of $1,789." +us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $210 standard deduction instead of $299, which raised net income to $1,853. It also used a $1,797 maximum allotment instead of $1,789. That gave $1,241/month instead of $1,259." +us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It returned $0 without doing any calculation. Gross income is 57% of the poverty guideline, the household is BBCE-eligible, and net income of $1,764 leaves a $1,259/month benefit against the $1,789 maximum for 8 people." +us,scenario_008,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 because it said there weren't enough inputs, but every needed input (household size, wages, self-employment income, interest) was given. The standard and earned income deductions give $1,764 net income and a $1,259/month benefit." +us,scenario_008,snap,gpt-5.5,llm_error,other,False,"It used the right deductions and maximum but didn't round the 30% contribution on $1,764 net income up from $529.20 to $530. That gave $1,260/month instead of $1,259." +us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $1,139/month is $120 short of $1,259, even though it said it applied a shelter deduction. That points to a maximum allotment well below the FY2026 $1,789. The correct result is $1,789 minus a $530 contribution, with no shelter deduction." +us,scenario_008,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $1,262/month is $3 above the correct $1,259. That matches a slightly overstated standard deduction or maximum allotment, plus skipping the rule that rounds the 30% contribution up to $530." +us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $1,284/month is $25 above the correct $1,259 ($1,789 maximum minus a $530 contribution on $1,764 net income). That is consistent with an overstated maximum allotment or standard deduction." +us,scenario_008,snap,gpt-6-astra,llm_error,other,False,"It computed on annual totals ($21,468 − 0.3 × $21,167) instead of monthly with rounding. The rules round net income to $1,764 and the contribution up to $530 each month, which gives $1,259/month and $15,108/year, not $15,117.90." +us,scenario_008,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $1,288/month is $29 above the correct $1,259. That is consistent with an 8-person maximum allotment above the FY2026 $1,789 or an overstated standard deduction." +us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"It had the right $1,763 net income and $1,789 maximum, but it rounded the 30% contribution down to $529. Net income rounds to $1,764, and the $529.20 contribution rounds up to $530, which gives $1,259/month." +us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It treated receiving housing assistance as disqualifying for SNAP. Housing assistance only affects the shelter deduction, which is $0 here anyway, so the household receives $1,259/month." +us,scenario_008,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied an excess medical deduction for the disabled spouse, but $300/year in over-the-counter costs is $25/month, below the $35 threshold. It also used a maximum allotment of about $1,822.50/month instead of $1,789." +us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"Its $1,772 net income implies a $291 standard deduction instead of $299. It also used a $1,759 maximum allotment instead of $1,789, which gave $1,227/month instead of $1,259." +us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used FY2025 parameters: a $1,756 maximum allotment and a $291 standard deduction. The FY2026 values are $1,789 and $299. It also worked on annual totals and skipped rounding the contribution up each month." +us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the $4,250 resource limit for households with a disabled member. New Jersey's TANF non-cash BBCE makes the household categorically eligible and removes the asset test, so the $69,500 in bank assets doesn't count." +us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"It used an 8-person maximum allotment of $1,806 instead of $1,789, and a $529 contribution instead of $530 (30% of $1,764, rounded up). That gave $1,276/month instead of $1,259." +us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no SNAP value and no explanation, so there was nothing to score." +us,scenario_008,snap,kimi-k3,llm_error,other,False,"It had the right $299 standard deduction, $513.33 earned income deduction and $1,789 maximum, but it skipped rounding. Net income rounds to $1,764 and the contribution rounds up to $530, which gives $1,259/month, not the fractional $1,259.825." +us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It applied a $3,750 asset limit for households with a disabled member. The household is categorically eligible through New Jersey's TANF non-cash BBCE, so there is no asset test and it receives $1,259/month." +us,scenario_008,snap,muse-wonder,llm_error,thresholds_rates,False,"It used an 8-person maximum allotment of $1,695 instead of $1,789. Its $1,772 net income implies about a $291 standard deduction instead of $299." +us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,795 maximum allotment instead of $1,789, and a $297 standard deduction instead of $299. That gave $1,265/month instead of $1,259." +us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $202 standard deduction instead of $299 and an $1,838 maximum instead of $1,789. It also left out interest income and then cut its own result without a stated reason because of housing assistance, which doesn't reduce SNAP here." +us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed $2,567/month exceeds 130% of the poverty guideline for 8 people. That limit is about $5,900/month, and gross income of $2,576 is only 57% of the guideline, so the household passes and receives $1,259/month." us,scenario_008,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Declared the spouse ineligible ""under adult NJ rules"" without running New Jersey's ACA expansion income test, which covers adults 19-64 up to 138% FPL and requires no categorical hook such as pregnancy or caretaker status. Household MAGI of about $30,900 against the eight-person FPL is 0.55 x FPL, far inside that limit; the bare denial is consistent with comparing the spouse's own $26,800 of wages to a one- or two-person 138% FPL line (roughly $22,600 to $30,700) instead of household MAGI over a family size of eight." us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to 0 by treating Medicaid eligibility as something requiring an affirmative listed trigger fact, extending the prompt's ""treat unlisted inputs as false"" rule to the computed output itself. Every input the determination needs was supplied - age 41, non-dependent, New Jersey, household income - and the MAGI adult-expansion test it never ran places household MAGI at 0.55 x FPL against the 138% FPL threshold, which returns eligible." us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"Ran the MAGI test correctly - about $31,420 of household income against the roughly $73,637 138%-FPL line for eight people - then reversed its own correct result with a fabricated rule that receipt of housing assistance disqualifies adults from Medicaid. Housing assistance is excluded from MAGI and disqualifies no one from any Medicaid category, and the MAGI adult category applies no resource test, so neither the housing subsidy nor the $69,500 in bank accounts affects eligibility." @@ -699,94 +710,96 @@ us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligib us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred breastfeeding or postpartum status and nutritional risk solely from the presence of a one-year-old, despite the prompt requiring unlisted statuses to be false. With neither pregnancy, breastfeeding, nor postpartum status listed, the spouse fails WIC's categorical test regardless of income." us,scenario_008,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly used disability as a WIC categorical qualifier. Disability and income eligibility do not establish WIC eligibility for an adult who is not pregnant, breastfeeding, or postpartum." us,scenario_008,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse two $1,000 exemptions, one for blind and one for disabled, for $13,000 total instead of $12,000. NJ allows one blind-or-disabled exemption per person. It then also subtracted an 18%-of-rent property-tax deduction ($1,251) that doesn't enter taxable income, so it taxed $16,664 instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Got the $12,000 in exemptions right but then subtracted a $1,250.82 renter property-tax deduction (18% of rent). The household takes the $50 refundable property-tax credit instead of the deduction, so taxable income stays at $18,915 and tax is $264.81, not $247.30." -us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Zeroed NJ tax by applying the federal American Opportunity Credit, a pension exclusion and a standard deduction, none of which reduce NJ Gross Income Tax here. NJ taxable income is $30,915 minus $12,000 in exemptions = $18,915, taxed at 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Said exemptions eliminate NJ tax without doing the math. The $12,000 in exemptions ($2,000 personal, $9,000 dependents, $1,000 blind/disabled) leave $18,915 of taxable income, which produces $264.81 at the 1.4% rate." -us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Computed about $251 after double-counting the spouse's blind and disabled exemptions. It then arbitrarily rounded the liability to $0 by citing nonrefundable credits that don't apply. The correct result is $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Misapplied the NJ $20,000 MFJ filing threshold: it said $30,915 of gross income was below it, when that income is above the threshold. With no exemption from tax, the $12,000 in exemptions leave $18,915 taxable at 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Used $1,500 personal exemptions for each spouse when NJ's regular exemption is $1,000 per filer ($2,000 for MFJ). That inflated exemptions to $13,000 and cut taxable income to $17,915 (tax $250.81) instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Gave the spouse separate $1,000 exemptions for blind and for disabled (NJ allows only one) to get about $251. It then invented a low-income phase-in that zeroes tax for MFJ filers above the $20,000 threshold, which NJ doesn't have. The correct tax is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Applied an 'NJ MFJ standard deduction' that wiped out income, but NJ has no standard deduction. Only exemptions ($12,000) reduce the $30,915, leaving $18,915 taxed at 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from a federal-style AGI of $30,632 by deducting half of SE tax, which NJ Gross Income doesn't allow, so it should start from $30,915. It also allowed only $9,000 of exemptions, valuing dependents at $1,000 instead of $1,500 each. That pushed taxable income into the 1.75% bracket ($21,632) instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Took a $181.70 NJ medical expense deduction using the $800 of over-the-counter health expenses above 2% of income. Nonprescription OTC items aren't qualifying medical expenses, so taxable income is $18,915 (tax $264.81), not $18,733.30." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Left out the spouse's $1,000 blind-or-disabled exemption and used $11,000 instead of $12,000, so it taxed $19,915 instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Got $18,915 of taxable income and about $265 of tax right, then subtracted the $50 NJ property-tax credit. That credit is refundable, so it doesn't reduce tax before refundable credits, which is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Claimed exemptions zero out NJ tax without computing them. $30,915 minus $12,000 in exemptions leaves $18,915 of taxable income and $264.81 of tax at 1.4%." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its preliminary $251 tax reflects $13,000 of exemptions: it gave the spouse separate blind and disabled exemptions, which NJ caps at one. It then subtracted the $50 property-tax credit, which is refundable and excluded from this output." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Used $14,000 of exemptions: $1,500 per spouse instead of the $1,000 regular exemption, plus two $1,000 exemptions for the spouse being both blind and disabled instead of one. That taxed $16,915 instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Answered $0 with no derivation. NJ income of $30,915 minus $12,000 in exemptions leaves $18,915 taxable, which produces $264.81 at 1.4%." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Computed $264.81 on $18,915 correctly, then subtracted the $50 NJ property-tax credit. That credit is refundable, so it isn't part of tax before refundable credits." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave separate $1,000 blind and disabled exemptions for the spouse, for $13,000 total. NJ allows a single blind-or-disabled exemption per person, so total exemptions are $12,000 and taxable income is $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no parseable value for state_income_tax_before_refundable_credits, so there is no substantive computation to grade." -us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Subtracted a made-up $4,000 MFJ standard deduction (NJ has none) and an 18%-of-rent property-tax deduction that isn't taken. It also undercounted exemptions at $9,000 by valuing dependents at $1,000 instead of $1,500, reaching $16,664 instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed NJ exemptions and credits erase the tax without computing them. $12,000 in exemptions leave $18,915 of taxable income, and no nonrefundable credit offsets the resulting $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Estimated $0 without computing the base. NJ gross income of $30,915 minus $12,000 in exemptions leaves $18,915, taxed at 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Took a $181.70 NJ medical deduction based on over-the-counter health expenses, which aren't qualifying medical expenses. Without it, taxable income is $18,915 and tax is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Used the NJ Child Tax Credit to offset the tax, but that credit is refundable and doesn't belong in tax before refundable credits. Tax on $18,915 at 1.4% is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted an OTC-based medical deduction that isn't allowed, then also netted the $50 renter property-tax credit, which is refundable. Both cuts are wrong. Tax before refundable credits is $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $250.81 reflects $13,000 of exemptions, counting separate blindness and disability exemptions for the spouse. NJ allows only one $1,000 blind-or-disabled exemption per person, so taxable income is $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Correctly excluded the refundable property-tax credit, but still took a $181.70 medical deduction from over-the-counter health expenses, which don't qualify. Taxable income is $18,915, not $18,733.30." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Computed $264.81 correctly, then subtracted the $50 renter property-tax credit. That credit is refundable, so it doesn't reduce tax before refundable credits." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Used a 'NJ standard deduction' that doesn't exist. Only the $12,000 in exemptions apply, leaving $18,915 taxable and $264.81 of tax." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Equated NJ gross income with federal AGI ($30,632), deducting half of SE tax, which NJ doesn't allow. It also valued dependents at $1,000 instead of $1,500 for $9,000 of exemptions, which pushed taxable income to $21,632 and into the 1.75% bracket." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Correctly picked the $50 property-tax credit over the deduction but subtracted it from tax. That credit is refundable and excluded from this output, which is $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Deducted $181.70 of NJ medical expenses from the $800 of over-the-counter health costs. Nonprescription OTC items aren't qualifying medical expenses, so taxable income stays at $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Left out the spouse's $1,000 blind-or-disabled exemption ($11,000 instead of $12,000). It also applied the 1.75% rate to all of its $19,915 even though NJ MFJ taxes the first $20,000 at 1.4%." -us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Understated exemptions at $10,000 instead of $12,000 ($2,000 personal + $9,000 dependents at $1,500 each + $1,000 blind/disabled). It then subtracted a $1,251 renter property-tax deduction that doesn't enter taxable income." -us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value for state_income_tax_before_refundable_credits, so there is no substantive computation to grade." -us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Computed $264.81 correctly, then subtracted the $50 renter property-tax credit. That credit is refundable and doesn't reduce tax before refundable credits." -us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Started from a federal-style AGI of $30,632, deducting half of SE tax, which NJ Gross Income doesn't allow. It also left out the spouse's $1,000 blind-or-disabled exemption, giving $19,632 of taxable income instead of $18,915." -us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Took a $181.70 medical deduction from over-the-counter health expenses, which don't qualify, and then subtracted the $50 renter property-tax credit, which is refundable. Neither belongs in the computation, which gives $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Valued dependents at $1,000 instead of $1,500 and got about $296. It then invented a 'nonrefundable exemption reduction' that zeroes the tax, which NJ doesn't have. With $12,000 of exemptions, tax is $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted that NJ taxable income is zero after exemptions, but $12,000 of exemptions against $30,915 of NJ gross income leaves $18,915 taxable, which produces $264.81." -us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It phased the federal EITC down to about $6,110, but earned income of about $30.5k is below the 2026 MFJ phase-out start, so the full $8,231 applies (NJ EITC $3,292.40). It then gave a $400-per-child NJ CTC based on gross-income brackets, although NJ taxable income is well under $30,000. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The NJ EITC is right at $3,292.40, but the model left out the $50 NJ property tax credit even though the household pays $6,949 in rent. It also used the superseded $1,000-per-child CTC ($2,000) instead of the June 30, 2026 schedule, which pays $2,500 for the two children age 5 and under." -us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed New Jersey has no refundable income tax credits. NJ's EITC (40% of federal, $3,292.40), child tax credit ($2,500) and $50 property tax credit are all refundable and all apply to this low-income family of eight." -us,scenario_008,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It said no NJ refundable credits apply. The household qualifies for the refundable NJ EITC ($3,292.40 = 40% of the $8,231 federal EITC), the NJ child tax credit for two children under 6 ($2,500), and the $50 property tax credit." -us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own working reached about $3,880: a phased-down EITC of about $7,200 instead of the full $8,231, plus a CTC for only the age-1 child, which misses the age-5 child. It then submitted an unrelated $9,492. The correct sum is $3,292.40 EITC + $2,500 CTC + $50 property tax credit." -us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The NJ EITC of $3,292 is right. The model then cut the CTC to about $1,200 by applying a $600 tier at roughly $30,500 of gross income, but the NJ CTC tier depends on NJ taxable income, which is far below $30,000 after exemptions. It filled the rest of its total with unexplained 'minor amounts' instead of the $50 property tax credit and the $2,500 CTC under the June 2026 schedule." -us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It used a federal EITC of about $7,700 instead of the $8,231 maximum, which is unreduced at this earned income. It also wrongly said the NJ child tax credit does not apply, when the two children age 5 and under qualify for $2,500 at this low NJ taxable income. It left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It correctly computed the $3,292.40 NJ EITC and the $50 property tax credit, but used the pre-2026 NJ CTC of $1,000 per child under 6 ($2,000). The schedule New Jersey enacted on June 30, 2026 for 2026–2028 pays $2,500 for these two young children at low taxable income." -us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used a stale $7,830 EITC maximum and a $25,511 phase-out start, which is not the MFJ threshold, and phased the federal EITC down to $6,716. For 2026 MFJ filers, $30.5k of earned income still earns the full $8,231 (NJ EITC $3,292.40). It also used $500 per young child instead of the taxable-income-based CTC and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It overstated the federal EITC at $8,000–$9,000, giving about $3,800 of NJ EITC, but the 3+ child maximum is $8,231, so the NJ EITC is $3,292.40. It used the old $2,000 CTC and left out the $50 property tax credit, so its total is close to the reference only because these errors offset." -us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied the $800 NJ CTC tier for $30,001–$40,000 using gross income. The NJ CTC is keyed to NJ taxable income, which is well under $30,000 after the personal, dependent and blind/disabled exemptions, and the correct amount is $2,500 for the two children under 6. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on a federal EITC of $8,021 instead of the $8,231 maximum. It left out the refundable NJ child tax credit ($2,500 for the two children age 5 and under) and the $50 property tax credit." -us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an understated $7,787 federal EITC instead of the full $8,231. It omitted the refundable NJ child tax credit ($2,500) and the $50 property tax credit." -us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used a federal EITC of $8,190 instead of the $8,231 2026 maximum and left out the $50 NJ property tax credit for this renter household. It also used the superseded $1,000-per-child CTC instead of the June 2026 schedule's $2,500 for two young children." -us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a federal EITC of $8,247 instead of $8,231, and a $75 'renters' credit' instead of the $50 NJ property tax credit. It also used the old $2,000 CTC instead of the $2,500 that the June 30, 2026 schedule pays for two children under 6." -us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used an understated $7,691 federal EITC instead of the unreduced $8,231. It put the NJ CTC in the $800 tier for $30,000–$40,000 based on gross income, although NJ taxable income is under $30,000 and the credit is $2,500. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It found no NJ refundable credits, missing the 40% NJ EITC ($3,292.40), the NJ child tax credit for two children under 6 ($2,500), and the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It took the $800-per-child CTC tier from gross income near $30,900. NJ taxable income after exemptions is far below $30,000, which yields $2,500 for the two children under 6. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It phased the federal EITC down to $7,128, but $30.5k of MFJ earned income is below the 2026 phase-out start, so the full $8,231 applies (NJ EITC $3,292.40). It also used the $800 gross-income CTC tier instead of the taxable-income-based $2,500, and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no computation, and $1,200 is less than half of the NJ EITC alone. The correct total is 40% × $8,231 = $3,292.40 EITC, plus $2,500 CTC for two children under 6, plus the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its NJ EITC of $3,219.60 is based on the 2025 federal maximum of about $8,046 instead of the 2026 maximum of $8,231. Its $1,600 CTC applies the $800 gross-income tier instead of the NJ taxable-income test that yields $2,500. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"$4,776 equals a $1,600 CTC ($800 tier, chosen from gross income) plus about $3,176 of NJ EITC from an understated federal EITC. The correct components are $3,292.40 EITC, $2,500 CTC based on NJ taxable income under $30,000, and the $50 property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are right. It put the NJ CTC in the $800 tier because gross income exceeds $30,000, but the tier is set by NJ taxable income, which is well below $30,000 after exemptions, and the credit for the two young children is $2,500." -us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so no substantive answer was given." -us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used an overstated $8,360 federal EITC and counted only one child under 6, missing that the 5-year-old also qualifies. It also gave $500 instead of the $2,500 total the CTC pays for two children at low NJ taxable income, and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It found no refundable NJ credit. A married family of eight with about $30.5k of earned income receives the 40% NJ EITC ($3,292.40), the NJ CTC for two children under 6 ($2,500), and the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It set refundable credits to zero because of 'no specific circumstances', but low-income status with qualifying children is exactly what triggers the NJ EITC ($3,292.40) and NJ CTC ($2,500), and rent triggers the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The $3,292.40 EITC is right, but the model left out the $50 NJ property tax credit even though $6,949 in rent is listed. It also applied the superseded $1,000-per-child CTC instead of the June 30, 2026 schedule, which pays $2,500 for the two children age 5 and under." -us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It based the NJ EITC on the 2025 federal maximum of $8,046 instead of the 2026 maximum of $8,231. It left out the refundable NJ child tax credit ($2,500 for two children under 6) and the $50 property tax credit entirely." -us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It gave $1,600 of CTC by using the $800 tier for incomes above $30,000, but the NJ CTC is keyed to NJ taxable income, which is well under $30,000 after exemptions, and the credit is $2,500. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,892.40 is the $3,292.40 EITC plus a $1,600 CTC from the $800 gross-income tier. The CTC is based on NJ taxable income below $30,000 and equals $2,500 for the two young children, and the model also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are right, but it used the $800-per-child CTC tier for $30,001–$40,000. NJ taxable income after exemptions is far below $30,000, so the CTC for the two children under 6 is $2,500." -us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied a 50% NJ EITC rate, but New Jersey's rate is 40% of the federal credit, giving $3,292.40 on the $8,231 federal EITC. It also used the old $2,000 CTC instead of $2,500 and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It overstated the NJ EITC at $3,337 instead of 40% × $8,231 = $3,292.40, and used the $800-per-child CTC tier from gross income instead of the taxable-income-based $2,500. It also left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It computed the $3,292.40 EITC and the $50 property tax credit correctly, but used the pre-2026 $1,000-per-child CTC ($2,000). The June 30, 2026 NJ schedule for 2026–2028 pays $2,500 for these two children under 6 at low taxable income." -us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It said no state refundable credits apply. The household receives the refundable NJ EITC ($3,292.40), the NJ CTC ($2,500) and the $50 property tax credit." -us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It placed the NJ CTC in a $30,001–$40,000 bracket using gross income and used only $400 per child. The credit depends on NJ taxable income, which is under $30,000, and equals $2,500 for the two young children. It also overstated the federal EITC at $8,247 instead of $8,231 and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $500-per-child NJ CTC instead of the $2,500 the credit pays for the two children under 6 at low NJ taxable income. It also overstated the federal EITC at $8,250 and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It explicitly excluded the $50 NJ property tax credit because it saw 'no inferred rent', but the spouse's $6,949 rent is listed. It also used the superseded $1,000-per-child CTC instead of the June 30, 2026 schedule's $2,500 for the two young children." -us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an understated $7,901 federal EITC instead of $8,231, and stated there were no other credits. It missed the refundable NJ CTC ($2,500) and the $50 property tax credit." -us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an $8,271 federal EITC instead of $8,231. It omitted the refundable NJ child tax credit for the two children under 6 ($2,500) and the $50 property tax credit." -us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so no substantive answer was given." -us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The $3,292.40 EITC is right, but the model left out the $50 NJ property tax credit that applies to this renter household. It also used the pre-2026 $1,000-per-child CTC instead of the June 30, 2026 schedule's $2,500 for two children under 6." -us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no NJ refundable credits apply, missing the 40% NJ EITC ($3,292.40), the NJ CTC ($2,500) and the $50 property tax credit, all of which this family qualifies for." -us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It phased down the NJ CTC because $30,915 of gross income exceeds $30,000. The CTC depends on NJ taxable income, which is far below $30,000 after exemptions, so no reduction applies and the credit is $2,500 under the 2026 schedule. It also overstated the federal EITC at $8,262 and left out the $50 property tax credit." -us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It claimed the federal EITC phases out to near $0 at $30,800 of earned income. For MFJ filers with 3+ children, that income is still at the $8,231 maximum, so the NJ EITC is $3,292.40. It also wrongly zeroed the NJ CTC ($2,500) and the $50 property tax credit." -us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It wrongly said the household is above an NJ EITC phase-out and invented a $100 minimum credit. The NJ EITC is simply 40% of the full $8,231 federal EITC ($3,292.40), and the NJ CTC ($2,500) and $50 property tax credit are also due." +us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse two separate $1,000 exemptions for blindness and disability, but NJ allows one per blind-or-disabled person. This made exemptions $13,000 instead of $12,000. It also subtracted an 18%-of-rent tenant property-tax deduction ($1,251), which does not apply here, ending at $16,664 taxable instead of $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Exemptions of $12,000 were correct. It then also subtracted a $1,250.82 renter property-tax deduction (18% of $6,949 rent), which is not available to this housing-assisted household. Taxable income came out at $17,664 instead of $18,915, giving $247.30 instead of $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Applied the federal American Opportunity Credit and an unspecified NJ exclusion to reach zero, but NJ has no AOC. The only reductions are $12,000 of exemptions, which leave $18,915 taxable at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Claimed exemptions eliminate the tax, but NJ exemptions here total only $12,000 against $30,915 of gross income. That leaves $18,915 taxable and $264.81 of tax." +us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Computed about $251 after double-counting the spouse's blind and disabled exemptions ($13,000 total instead of $12,000), then rounded to $0 citing unnamed credits. No nonrefundable NJ credit applies, so the tax on $18,915 stays at $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Called $30,900 of gross income 'below' the $20,000 MFJ filing threshold, which is backwards. The household is over the threshold, and after $12,000 of exemptions it owes 1.4% on $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Used $1,500 personal exemptions for each spouse ($3,000), but NJ's regular exemption is $1,000 per taxpayer ($2,000 for MFJ). This overstated exemptions by $1,000 and gave $17,915 taxable instead of $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Double-counted the spouse's blind and disabled exemptions ($13,000 total instead of $12,000) and got about $251. It then invented a low-income phase-in that zeroes the liability, but NJ has no such provision above the $20,000 MFJ threshold." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Applied an 'NJ MFJ standard deduction' that does not exist. NJ uses only exemptions, which total $12,000 here and leave $18,915 taxable." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $30,632, which subtracts half of SE tax, but NJ gross income allows no such adjustment and is $30,915. It also used only $9,000 of exemptions, valuing dependents at $1,000 instead of $1,500, which pushed taxable income into the 1.75% bracket." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Exemptions were correct at $12,000, but it subtracted a $181.70 medical deduction built from over-the-counter health spending. OTC expenses are not qualifying NJ medical expenses, so taxable income stays at $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Used $11,000 of exemptions and left out the $1,000 blind-or-disabled exemption the spouse qualifies for. Taxable income came out at $19,915 instead of $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Correctly reached $265 on $18,915, then subtracted the $50 NJ tenant property-tax credit as nonrefundable. That credit is refundable (and not applied to this housing-assisted household), so it cannot reduce tax before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Asserted that exemptions eliminate the tax without computing it. $12,000 of exemptions against $30,915 leaves $18,915 taxable, or $264.81 at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its preliminary $251 reflects $13,000 of exemptions, so it double-counted the spouse's blind and disabled exemptions. It then subtracted the $50 tenant property-tax credit, which is refundable and excluded from this output." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Claimed $14,000 of exemptions. It used $1,500 per spouse instead of $1,000 ($3,000 vs $2,000) and gave separate blind and disabled exemptions ($2,000 vs $1,000), so it overstated exemptions by $2,000." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Reported zero with no computation. NJ gross income of $30,915 minus $12,000 of exemptions leaves $18,915 taxable at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Correctly got $264.81 on $18,915, then subtracted the $50 NJ property-tax credit. That credit is refundable (and not available to this housing-assisted household), so it does not reduce tax before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Used $13,000 of exemptions by giving the spouse separate $1,000 exemptions for blindness and disability. NJ allows one blind-or-disabled exemption per person, so taxable income is $18,915, not $17,915." +us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no value or reasoning for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Valued dependents at $1,000 instead of $1,500 ($9,000 of exemptions instead of $12,000) and subtracted a $4,000 MFJ standard deduction that NJ does not have. It also took an 18%-of-rent property-tax deduction that does not apply to this household." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed exemptions and credits erase the tax without computing it. $12,000 of exemptions leaves $18,915 taxable at 1.4%, and no nonrefundable NJ credit applies." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Estimated zero without computing. NJ gross income of $30,915 less $12,000 of exemptions gives $18,915 taxable and $264.81 of tax." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Exemptions were correct at $12,000, but it subtracted a $181.70 medical deduction from over-the-counter health spending. That spending does not qualify for the NJ medical deduction, which lowered taxable income below the correct $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Used the NJ Child Tax Credit to offset the tax, but that credit is refundable and belongs outside this pre-refundable output. Tax before refundable credits is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted a medical deduction built from OTC health spending, which is not a qualifying NJ medical expense. It then took off the $50 renter property-tax credit, which is refundable and excluded here." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $250.81 equals 1.4% of $17,915, which means it gave separate blindness and disability exemptions ($13,000 total). NJ allows one $1,000 blind-or-disabled exemption per person, so the total is $12,000." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Exemptions were correct at $12,000, but it subtracted an estimated $181.70 medical deduction from over-the-counter health spending. That spending does not count toward the NJ medical deduction, so taxable income is $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Correctly reached $264.81 on $18,915, then subtracted a $50 renter property-tax credit. That credit is refundable (and not applied to this housing-assisted household), so it does not belong in tax before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Applied an NJ standard deduction that does not exist. NJ allows only exemptions, which total $12,000 here and leave $18,915 taxable." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used federal AGI of $30,632, which subtracts half of SE tax, instead of NJ gross income of $30,915. It also valued dependents at $1,000 instead of $1,500 ($9,000 of exemptions instead of $12,000), which wrongly pushed income into the 1.75% bracket." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Correctly got $265 on $18,915, then subtracted the $50 renter property-tax credit as nonrefundable. That credit is refundable (and not applied to this housing-assisted household), so pre-refundable tax stays at $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Exemptions were correct at $12,000, but it treated the $800 of over-the-counter health spending as deductible medical expense above 2% of income ($181.70). OTC spending does not qualify for the NJ medical deduction." +us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Left out the spouse's $1,000 blind-or-disabled exemption ($11,000 instead of $12,000). It then applied 1.75% to all of $19,915, but the MFJ schedule taxes the first $20,000 at 1.4%." +us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used only $10,000 of exemptions, undercounting the $1,500-per-dependent amount (the correct total is $12,000). It also subtracted a $1,251 renter property-tax deduction that does not apply to this housing-assisted household." +us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or reasoning for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Correctly computed $264.81 on $18,915, then subtracted the $50 renter property-tax credit. That credit is refundable (and not applied to this housing-assisted household), so it does not reduce tax before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Used federal AGI of $30,632, which subtracts half of SE tax that NJ does not allow, instead of NJ gross income of $30,915. It also left out the spouse's $1,000 blind-or-disabled exemption, using $11,000 instead of $12,000." +us,scenario_008,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Used $13,000 of exemptions by counting the spouse's blindness and disability as two separate $1,000 exemptions. NJ allows only one blind-or-disabled exemption per person, so taxable income is $18,915, not $17,915." +us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Subtracted a $181.70 medical deduction from OTC health spending, which is not a qualifying NJ medical expense. It then took off the $50 renter property-tax credit, which is refundable and excluded from tax before refundable credits." +us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Valued dependents at $1,000 instead of $1,500, understating exemptions. It then invented a 'nonrefundable exemption reduction' that zeroes the tax, but no such NJ provision exists; correct exemptions of $12,000 leave $18,915 taxable." +us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed NJ taxable income is zero, but exemptions total only $12,000 against $30,915 of gross income. That leaves $18,915 taxable at 1.4% = $264.81." +us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Cut the federal EITC to about $6,110 even though earned income sits at the phase-out start; the full $8,231 applies, so the NJ EITC is $3,292.40. It gave $400 per child under 6 by testing gross income, but NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $2,500 for the two children. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but the NJ CTC used the old $1,000-per-child amount instead of the June 30, 2026 schedule, which pays $2,500 for the two children age 5 and under. It also left out the $50 NJ property tax credit that renters get, even though $6,949 in rent is listed." +us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Claimed New Jersey has no refundable credits. In fact the NJ EITC (40% of federal, $3,292.40), the NJ Child Tax Credit ($2,500) and the $50 property tax credit are all refundable and all apply to this family." +us,scenario_008,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Reported zero, missing the refundable NJ EITC ($3,292.40 = 40% of the $8,231 federal EITC), the $2,500 NJ CTC for the two children age 5 and under, and the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own pieces add up to about $3,880 ($2,880 EITC plus $1,000 CTC), yet it submitted an unexplained $9,492. It also understated the federal EITC (the full $8,231 applies), counted only the age-1 child for the CTC when the age-5 child also qualifies, and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The EITC is correct, but it tested the NJ CTC against gross income near $30,500 and gave $600 per child. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $2,500 total. It then padded the total with about $892 of unspecified 'minor amounts' instead of the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Said the NJ CTC does not apply at this income, but NJ taxable income is under $30,000 and the two children age 5 and under get $2,500. It also used a federal EITC of about $7,700 instead of the full $8,231 and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct. The only miss is the CTC: it applied the old $1,000-per-child schedule instead of the schedule enacted June 30, 2026 for tax years 2026–2028, which pays $2,500 for the two children under 6." +us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Used outdated EITC figures (a $7,830 maximum and a phase-out starting near $25,500–$26,149) and reduced the federal credit to $6,716. The 2026 maximum of $8,231 applies in full here. It also used the outdated $500-per-child NJ CTC instead of the $2,500 total under the 2026 schedule and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Overstated the federal EITC as $8,000–$9,000 and the NJ EITC as about $3,800; the correct figures are $8,231 and $3,292.40. It applied the old $1,000-per-child CTC ($2,000) instead of $2,500 under the 2026 schedule and left out the $50 property tax credit. Its total comes near the reference only because these errors partly cancel." +us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The EITC is correct, but it put the household in the $30,001–$40,000 NJ CTC bracket based on gross income. NJ taxable income after the personal and $1,500-per-dependent exemptions is under $30,000, and the 2026 schedule pays $2,500. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Counted only the NJ EITC, using an understated $8,021 federal EITC instead of $8,231. It left out the refundable $2,500 NJ Child Tax Credit for the two children age 5 and under and the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Counted only the NJ EITC, using an understated $7,787 federal EITC (the correct figure is $8,231, giving $3,292.40). It left out the $2,500 refundable NJ Child Tax Credit and the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Used a federal EITC of $8,190 instead of $8,231 and applied the old $1,000-per-child NJ CTC ($2,000) instead of $2,500 under the 2026 schedule. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied the old $1,000-per-child NJ CTC ($2,000) instead of $2,500 under the 2026 schedule. It also replaced the $50 NJ property tax credit with a nonexistent $75 'renters' credit' and used a federal EITC of $8,247 instead of $8,231." +us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Understated the federal EITC at $7,691 (the full $8,231 applies) and put the CTC in the $800 bracket by testing gross income instead of NJ taxable income, which is under $30,000. That missed the $2,500 under the 2026 schedule, and it also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Found no NJ refundable credits, missing the NJ EITC ($3,292.40), the $2,500 NJ Child Tax Credit for the two children age 5 and under, and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Gave $800 per child under 6 by testing gross income against the $30,000 cutoff. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $2,500. It also left out the $50 property tax credit and slightly understated the EITC ($3,280 vs $3,292.40)." +us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Reduced the federal EITC to $7,128 when the full $8,231 applies, giving an NJ EITC of $2,851 instead of $3,292.40. It used the $800 CTC bracket instead of $2,500 under the 2026 schedule for taxable income under $30,000 and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Submitted $1,200 with no calculation. That is less than the NJ EITC alone (40% × $8,231 = $3,292.40), and it leaves out the $2,500 NJ CTC and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Understated the NJ EITC at $3,219.60 instead of $3,292.40 and gave the CTC at the $800-per-child bracket ($1,600) instead of $2,500 under the 2026 schedule for taxable income under $30,000. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,776 matches an understated NJ EITC plus a $1,600 CTC at the $800 bracket. The correct figures are a $3,292.40 EITC, a $2,500 CTC under the 2026 schedule for taxable income under $30,000, and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are correct, but it put the CTC in the $800 bracket by testing gross income. NJ taxable income after exemptions is under $30,000, so the 2026 schedule pays $2,500 rather than $1,600." +us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no parseable value for state_refundable_credits. The correct total is $3,292.40 NJ EITC + $2,500 NJ CTC + $50 property tax credit." +us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Counted only one child under 6, but the NJ CTC covers children age 5 and under, so both the 5-year-old and the 1-year-old qualify. It used an outdated $500 amount instead of the 2026 schedule ($2,500 total), overstated the federal EITC at $8,360, and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Reported zero, missing the refundable NJ EITC ($3,292.40 = 40% of the $8,231 federal EITC for three or more children), the $2,500 NJ CTC, and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Reported zero, but the low-income working family with six children qualifies for the NJ EITC ($3,292.40), the $2,500 NJ Child Tax Credit for the two children under 6, and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The EITC is correct, but it applied the old $1,000-per-child NJ CTC instead of the June 30, 2026 schedule ($2,500 for the two children under 6). It also left out the $50 property tax credit that renters get, even though $6,949 in rent is listed." +us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Counted only the NJ EITC, using an understated $8,046 federal EITC instead of $8,231. It left out the refundable $2,500 NJ Child Tax Credit and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The EITC is correct, but it gave only $1,600 in CTC ($800 per child), a bracket that applies above $30,000 of income. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $2,500. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,892.40 combines the correct $3,292.40 EITC with a $1,600 CTC from the $800 bracket. The household's NJ taxable income is under $30,000, so the 2026 schedule pays $2,500, and it also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are correct, but it gave a $1,600 CTC from the $800 bracket. NJ taxable income after the $1,500-per-dependent exemptions is under $30,000, and the 2026 schedule pays $2,500." +us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Applied a 50% NJ EITC rate, but the statutory rate is 40%, so the credit is $3,292.40 rather than $4,081. It used the old $1,000-per-child CTC instead of $2,500 under the 2026 schedule and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Overstated the NJ EITC at $3,337 instead of $3,292.40 and gave $800 per child from the bracket above $30,000. NJ taxable income is under $30,000, so the 2026 schedule pays $2,500. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct. The only miss is the CTC: it used the old $1,000-per-child amount instead of the schedule enacted June 30, 2026, which pays $2,500 for the two children under 6." +us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no NJ refundable credits apply, missing the NJ EITC ($3,292.40), the $2,500 NJ Child Tax Credit and the $50 property tax credit." +us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Put the household in the $30,001–$40,000 CTC bracket based on gross income and paid only $400 per child. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $2,500. It also left out the $50 property tax credit and slightly overstated the EITC ($3,299)." +us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used an outdated $500-per-child NJ CTC instead of the 2026 schedule, which pays $2,500 for the two children under 6. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Applied the old $1,000-per-child CTC instead of the June 30, 2026 schedule ($2,500 total). It also excluded the $50 property tax credit, saying rent was not given, when the spouse's $6,949 rent is listed." +us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Said there are no other state refundable credits, leaving out the $2,500 NJ Child Tax Credit for the two children age 5 and under and the $50 property tax credit. It also understated the federal EITC at $7,901 instead of $8,231." +us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Counted only the NJ EITC, using $8,271 instead of the $8,231 federal maximum. It left out the $2,500 refundable NJ Child Tax Credit and the $50 property tax credit." +us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value for state_refundable_credits. The correct total is $3,292.40 NJ EITC + $2,500 NJ CTC + $50 property tax credit." +us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The EITC is correct, but it applied the old $1,000-per-child CTC instead of the 2026 schedule, which pays $2,500 for the two children under 6. It also left out the $50 property tax credit that renters get, even though rent is listed." +us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Reported zero, missing the NJ EITC ($3,292.40), the $2,500 NJ Child Tax Credit and the $50 property tax credit, all of which are refundable and apply to this family." +us,scenario_008,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Understated the federal EITC at $8,046 instead of $8,231 and applied the old $1,000-per-child CTC instead of $2,500 under the 2026 schedule. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Reduced the CTC with an invented linear phase-down based on $915 of gross income above $30,000. The NJ CTC uses NJ taxable income, which is under $30,000 after exemptions, and the 2026 schedule pays $2,500. It also left out the $50 property tax credit and overstated the federal EITC at $8,262." +us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Claimed the federal EITC phases out to about $0 at $30,800 of earned income. For married filing jointly with three or more children, that income is at the phase-out start, so the full $8,231 applies and the NJ EITC is $3,292.40. It also dropped the $2,500 NJ CTC and the $50 property tax credit, wrongly treating housing assistance as cancelling them." +us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Invented a separate NJ EITC phase-out and a $100 minimum credit. The NJ EITC is simply 40% of the full $8,231 federal EITC ($3,292.40). It also left out the $2,500 NJ CTC and the $50 property tax credit." us,scenario_008,tanf,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model submitted a bare number with no derivation; the correct derivation applies New Jersey's WFNJ eligibility standard and yields $0, because countable income of about $2,576/month exceeds the family-of-eight standard (roughly $770/month) even after the 50% earned-income disregard, and $69,500 in bank assets exceeds the $2,000 TANF resource limit. Its $5,800 figure (about $483/month) is consistent with pulling a large-family NJ grant-schedule amount and shading it down for earnings — treating TANF as a residual gap-filler for a household with six children — while never running the income eligibility gate or the asset test at all." us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got exactly the right result: AGI $105,900, minus the $32,200 standard deduction, gives $73,700, taxed at $2,480 + $5,868 = $8,348. It then submitted 3,455, a number that doesn't follow from any of its own work, so the error is an output mismatch rather than a tax-rule error." us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,139 ESI premium from wages that are already gross W-2 income. It also counted all $30,000 of Social Security rather than the $25,500 taxable under the 85% cap, and used the 2025 $30,000 standard deduction instead of the 2026 $32,200. Its own estimate was about $8,600, yet it submitted $2,950 with no derivation." @@ -883,43 +896,44 @@ us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,age_disability,False, us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,age_disability,False,"The model based eligibility only on household income being under the WIC limit and skipped the categorical requirement. A child qualifies for WIC only before the 5th birthday, so a 10-year-old is ineligible." us,scenario_012,child1_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model wrongly said that age 10 is within WIC's age limit. WIC child eligibility ends at the 5th birthday, so Child 1 fails the categorical test even though household income is below the income limit (185% of the federal poverty line)." us,scenario_012,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly found EITC $4,427 + ACTC $1,700 = $6,127. It then added an unexplained $966 (it said the pregnancy does not add a qualifying child) and submitted $7,093. That is an arithmetic error added on top of the correct derivation." -us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted the pregnant spouse/unborn child as qualifying children and claimed three, but only the 10-year-old qualifies for EITC or CTC. It also used a head-of-household figure of $3,995 instead of the $4,427 joint one-child maximum for 2026, and it left out the $1,700 refundable ACTC entirely." -us,scenario_012,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model claimed the EITC is about $0 at this income, but $22,000 is on the one-child plateau and earns the full $4,427. It also said the refundable CTC does not apply, even though zero tax liability and $19,500 of earnings above $2,500 produce a $1,700 ACTC." -us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted $4,427. It then added an arbitrary inflation adjustment to reach $6,172 instead of $4,427 + $1,700 = $6,127. The $1,700 ACTC was correct." -us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model replaced its ~$4,400 EITC estimate with a 'refined' one-child maximum of $4,731. The actual 2026 maximum is $4,427, so the EITC was overstated by $304." -us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model capped the ACTC at $1,800 per child, but the 2026 refundable cap is $1,700. It also said the EITC was near phase-out (~$4,367), when $22,000 is below the joint phase-out start and gets the full $4,427." -us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model kept the 2025 EITC maximum of $4,328 instead of the 2026 value of $4,427. It also treated the whole $2,000 CTC as refundable, ignoring the $1,700 per-child refundable cap on the ACTC." -us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used stale one-child figures ($3,995 plateau, $11,610 phase-in) and wrongly put $22,000 in the phase-out range. It then dropped that work and submitted an unsupported ~$5,500 EITC; the correct full 2026 one-child EITC is $4,427, which with the $1,700 ACTC totals $6,127." -us,scenario_012,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used $4,404 as the 2026 one-child EITC maximum instead of the indexed $4,427. That understated the total by $23." -us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the pre-TCJA sunset rules: a $1,000 CTC with a $3,000 earnings threshold. Under OBBBA, the CTC is permanent with a $1,700 refundable cap and a $2,500 threshold. The model also understated the 2026 one-child EITC as $4,355 instead of $4,427." -us,scenario_012,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model limited the ACTC to the post-sunset $1,000, but OBBBA's 2026 refundable cap is $1,700. It also overstated the one-child EITC maximum as $4,442 instead of $4,427." -us,scenario_012,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model overstated both parts. It used an EITC of $4,454 instead of the 2026 one-child maximum of $4,427, and an ACTC of $1,749 when the 2026 refundable cap is exactly $1,700 per child." -us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the expired $1,000 refundable CTC instead of OBBBA's 2026 refundable cap of $1,700. It also put the EITC at $4,423 instead of $4,427." -us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model put the one-child EITC at ~$5,571, well above the 2026 one-child maximum of $4,427. It also used the sunset-era $1,000 refundable CTC instead of the $1,700 OBBBA refundable cap." -us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model rounded the EITC to $4,500 instead of the exact 2026 one-child maximum of $4,427. It also used a $1,000 refundable CTC instead of the 2026 ACTC cap of $1,700." -us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used ~$4,320 for the one-child EITC instead of the 2026 maximum of $4,427. It also used a post-sunset $1,000 refundable CTC instead of the $1,700 OBBBA refundable cap." -us,scenario_012,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $4,050 is below the one-child EITC maximum alone ($4,427), so it understated the EITC and dropped all or nearly all of the $1,700 refundable ACTC that brings the correct total to $6,127." -us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted $4,427, which understated the total by $99." -us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 indexed $4,427, which understated the total by $99." -us,scenario_012,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum of $4,328 instead of the 2026 value of $4,427. It also used a $1,000 ACTC from the TCJA sunset instead of the permanent OBBBA $1,700 refundable cap." -us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model applied a reduction that cut the EITC to $2,591, even though $22,000 is past the phase-in end and below the joint phase-out start, so the full $4,427 applies. It also got refundability backwards: it said the ACTC is $0 because there is no tax liability, when zero liability is exactly what makes the full $1,700 refundable." -us,scenario_012,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 OBBBA CTC as refundable, but the 2026 refundable portion (ACTC) is capped at $1,700 per child. It also put the EITC at $4,420 instead of $4,427." -us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The model denied the EITC because the spouse is 18, but the 25–64 age test applies only to filers without qualifying children. A joint return with a qualifying 10-year-old gets the full $4,427 regardless of the spouse's age. Its $2,524 refundable CTC also exceeds the $1,700 per-child ACTC cap." -us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed filing status and earned income were unspecified and returned $0, but the facts give $22,000 of wages, a married couple, and a 10-year-old child. That supports a joint one-child EITC of $4,427 plus a $1,700 ACTC." -us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated both parts. It used $4,526 for the 2026 one-child EITC instead of $4,427, and an $1,800 ACTC instead of the $1,700 refundable cap." -us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The EITC of $4,427 was correct, but the model put the refundable CTC at $1,800 instead of the 2026 per-child refundable cap of $1,700, which overstated the total by $100." -us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model applied a phase-out that cut the EITC to $2,479, even though $22,000 is on the one-child plateau and gets the full $4,427. It also left out the $1,700 refundable ACTC." -us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the post-sunset ACTC ($1,000 cap, $3,000 threshold), but OBBBA made the CTC permanent with a $1,700 refundable cap in 2026. It also used $4,318 for the EITC maximum instead of $4,427." -us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the post-sunset $1,000 ACTC maximum instead of the OBBBA 2026 refundable cap of $1,700. It also overstated the one-child EITC as $4,436 instead of $4,427." -us,scenario_012,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used $4,437 as the 2026 one-child EITC maximum instead of $4,427, which overstated the total by $10." -us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC (sunset rules, $3,000 threshold) instead of OBBBA's $1,700 2026 refundable cap. It also projected the EITC at $4,436 instead of the published $4,427." -us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no reasoning for federal_refundable_credits, so there is no answer to score against the $6,127 EITC-plus-ACTC total." -us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model used $4,401 as the 2026 one-child EITC maximum instead of $4,427, which understated the total by $26." -us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model said the EITC and refundable CTC 'partially offset' and returned $0, but the two credits add together. $22,000 of earnings with one qualifying child gets the full $4,427 EITC plus a $1,700 ACTC." -us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The ACTC of $1,700 was correct, but the model approximated the 2026 one-child EITC maximum as $4,425 instead of $4,427, which left the total $2 short." -us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a second qualifying child, but only the 10-year-old qualifies for EITC or CTC. That led it to apply a two-child EITC of $7,830 and two $1,700 ACTCs, instead of the one-child $4,427 EITC plus one $1,700 ACTC." -us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model gave a one-child EITC of $6,936, above the 2026 one-child maximum of $4,427. It also set the ACTC to $0 on the claim that regular tax absorbs the CTC, but $22,000 of AGI is below the joint standard deduction, so tax is $0 and the full $1,700 is refundable." +us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly derived EITC $4,427 + ACTC $1,700 = $6,127. It then added an unsupported extra $966, labeled as a 'projected adjustment', and reported $7,093. No refundable credit makes up that $966." +us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It counted 3 qualifying children, including the pregnant spouse/unborn child, although only the 10-year-old qualifies. It used an outdated $3,995 EITC figure instead of the 2026 one-child maximum of $4,427. It also left out the $1,700 refundable ACTC." +us,scenario_012,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It claimed the EITC is roughly $0 for an MFJ one-child family at $22,000. That earnings level is past the phase-in end and below the phase-out start, so the full $4,427 applies. It also wrongly dropped the $1,700 ACTC, which is refundable precisely because tax liability is zero." +us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It started from the 2025 one-child EITC maximum of $4,328 and then made an arbitrary upward 'inflation' change to reach $6,172. The 2026 maximum is $4,427, so the total should be $4,427 + $1,700 = $6,127." +us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It replaced its own roughly $4,400 EITC estimate with an invented '2026 refined' maximum of $4,731. The 2026 one-child maximum is $4,427, so the EITC was overstated by $304. The $1,700 ACTC was correct." +us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It capped the ACTC at $1,800 per child, but the 2026 refundable limit is $1,700. It also used $4,367 for the EITC instead of the 2026 one-child maximum of $4,427." +us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It treated the full $2,000 CTC as refundable ACTC, but the ACTC is capped at the $1,700 refundable limit. It also used the 2025 EITC maximum of $4,328 instead of the 2026 maximum of $4,427." +us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Its reasoning flips between several approaches: it uses an outdated $3,995 plateau, wrongly puts $22,000 in the phase-out range, and then picks an arbitrary $5,500 EITC. The one-child EITC at $22,000 MFJ is the full 2026 maximum of $4,427, which with the $1,700 ACTC gives $6,127, not $7,200." +us,scenario_012,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The structure was right (full one-child EITC plus $1,700 ACTC), but it used $4,404 for the 2026 maximum EITC instead of $4,427 (34% × $13,020 earned-income amount)." +us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used post-TCJA-sunset ACTC rules (a $1,000 cap and a $3,000 threshold), but OBBBA made the TCJA CTC permanent, with a $1,700 refundable cap and a $2,500 threshold in 2026. It also used a $4,355 EITC instead of $4,427." +us,scenario_012,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the pre-TCJA $1,000 ACTC, but the 2026 refundable CTC is $1,700 per child. It also overstated the one-child EITC at $4,442; the correct figure is $4,427." +us,scenario_012,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It inflated both parameters. It used $4,454 for the EITC instead of the 2026 one-child maximum of $4,427, and $1,749 for the ACTC instead of the 2026 refundable cap of $1,700." +us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the $1,000 post-sunset refundable CTC instead of the 2026 ACTC cap of $1,700 under the permanent TCJA/OBBBA rules. It also put the EITC at $4,423 instead of $4,427." +us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It overstated the one-child EITC at $5,571; the 2026 maximum is $4,427. It also used a $1,000 refundable CTC instead of the $1,700 ACTC cap." +us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a rounded $4,500 EITC instead of the 2026 one-child maximum of $4,427. It also applied the $1,000 post-sunset refundable CTC instead of the 2026 ACTC cap of $1,700." +us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used roughly 2025-level EITC figures ($4,320) instead of the 2026 one-child maximum of $4,427. It also applied a $1,000 refundable CTC instead of the $1,700 ACTC cap." +us,scenario_012,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no breakdown. The correct total is the $4,427 maximum one-child EITC plus the $1,700 ACTC. Its $4,050 is short by $2,077, which is consistent with counting only an understated EITC and leaving out the refundable ACTC." +us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted maximum of $4,427. Its $1,700 ACTC was correct." +us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used the 2025 one-child EITC maximum of $4,328 instead of the 2026 maximum of $4,427. Its $1,700 ACTC was correct." +us,scenario_012,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 EITC maximum of $4,328 instead of the 2026 maximum of $4,427. It also used a $1,000 ACTC instead of the 2026 refundable cap of $1,700." +us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It reduced the EITC to $2,591, but $22,000 is below the MFJ one-child phase-out start, so the full $4,427 applies. It set the ACTC to $0 because there was no tax liability, which reverses the rule: with zero liability the ACTC is min($1,700, 15% × ($22,000 − $2,500)) = $1,700." +us,scenario_012,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It treated the full $2,200 CTC as refundable, but the 2026 refundable portion is capped at $1,700 per child. It also used a slightly low EITC of $4,420 instead of $4,427." +us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It ruled out the EITC because the spouse is 18. For a joint filer with a qualifying child there is no minimum-age test, so the full $4,427 one-child EITC applies. Its $2,524 ACTC also exceeds the $1,700 per-child refundable cap." +us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0, saying filing status and earned income were not specified. The facts give $22,000 of wages, a joint household, and a 10-year-old qualifying child, which produce a $4,427 EITC and a $1,700 ACTC." +us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It overstated the 2026 one-child EITC at $4,526 instead of $4,427. It also used an $1,800 ACTC cap instead of the 2026 refundable limit of $1,700." +us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,427 EITC was correct, but it used an $1,800 refundable CTC. The 2026 ACTC cap is $1,700 per child, so the total is $6,127." +us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It applied phase-out and cut the EITC to $2,479, but $22,000 is below the 2026 MFJ one-child phase-out start, so the full $4,427 applies. It also left out the $1,700 refundable ACTC." +us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used post-sunset ACTC rules ($1,000 cap, $3,000 threshold), but OBBBA made the TCJA CTC permanent, with a $1,700 refundable cap in 2026. It also used $4,318 for the EITC instead of the 2026 maximum of $4,427." +us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and capped the ACTC at $1,000, but the 2026 refundable CTC is $1,700 under OBBBA. It also put the EITC at $4,436 instead of $4,427." +us,scenario_012,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The structure and the $1,700 ACTC were right, but it used $4,437 for the 2026 one-child EITC maximum. The correct figure is $4,427 (34% × $13,020)." +us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000 refundable CTC instead of the 2026 ACTC cap of $1,700. It also inflated the 2025 EITC to $4,436 instead of the 2026 maximum of $4,427." +us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer or reasoning. The correct derivation is a $4,427 one-child EITC plus a $1,700 ACTC, for $6,127." +us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its $1,700 ACTC was correct, but it used $4,401 for the 2026 one-child EITC maximum instead of $4,427." +us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It returned $0 with no computation. At $22,000 of earnings with one qualifying child, the household gets the full $4,427 EITC plus a $1,700 refundable ACTC, and these are both added to refundable credits rather than offsetting each other." +us,scenario_012,federal_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Its $4,427 EITC was correct, but it used an $1,800 refundable CTC. The 2026 ACTC cap is $1,700 per child." +us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The method was right (full EITC plus $1,700 ACTC), but it approximated the 2026 one-child EITC maximum as $4,425. The correct figure is $4,427 (34% × $13,020 earned-income amount, rounded)." +us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It counted the unborn child as a second qualifying child, but only the 10-year-old qualifies. That doubled the ACTC to $3,400 and used a $7,830 multi-child EITC instead of the $4,427 one-child maximum." +us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It said the CTC was fully used against regular tax, but $22,000 of AGI is below the MFJ standard deduction, so liability is zero and the full $1,700 ACTC is refundable. Its $6,936 EITC also exceeds the one-child maximum of $4,427." us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said the facts did not support eligibility and never ran either qualifying test. At $22,000 for a family of three, income is 81% of the poverty guideline, under the 130% free-meal limit. The household's SNAP receipt of about $407.90 a month separately makes the child categorically eligible, so the correct tier is FREE (1), not 0." us,scenario_012,free_school_meals_eligible,grok-4.7,llm_error,age_disability,False,"The model applied the rule that unlisted booleans are false to K-12 enrollment and set it to false. PolicyEngine does not take enrollment as an input here; it derives it from age, so a 10-year-old counts as a K-12 student. With enrollment in place, SNAP categorical eligibility and income at 81% of the poverty guideline both put the child in the FREE tier." us,scenario_012,free_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model claimed income was above 130% of the poverty guideline, but $22,000 for a three-person household is only 81% of the guideline, well below the roughly $35,000 limit for that size. It also missed that SNAP receipt of about $407.90 a month makes the child categorically eligible for free meals whatever the income." @@ -935,52 +949,53 @@ us,scenario_012,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output us,scenario_012,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling as sufficient for reduced-price eligibility without first assigning the household to the free-meals tier. The household's 0.81 FPG ratio and categorical eligibility produce FREE status, which makes reduced-price eligibility false." us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that income below 130% FPG places the child in the free-meal range and that reduced-price support applies only from 130% through 185%, but then submitted 1 contrary to its own derivation. FREE-tier classification, independently reinforced by categorical eligibility, makes the reduced-price output 0." -us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the household's expected contribution ($378, which is 30% of net income) as the monthly benefit, instead of subtracting it from the $785 maximum to get $407. Its working also wrongly added a utility allowance (about $400) because of the energy subsidy, and used a $1,023 maximum allotment." -us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"The deductions, the $785 maximum and the 'no shelter deduction' finding were all right, but the model skipped SNAP's rounding rules. Rounding 30% of the $1,258 net income up gives a $378 contribution and a whole-dollar benefit of $407. Using the unrounded $407.70 inflated the annual total to $4,892." -us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left out the 20% earned-income deduction ($366.67) and used wrong amounts: a $256 standard deduction instead of $209 and a $694 maximum allotment instead of $785. It then treated the $411 energy subsidy as something that raises income, which cut the monthly benefit to $84 instead of $407." -us,scenario_012,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model wrongly found the household over the income limits, partly by counting four people. Gross income of $1,833 is far below the 130% limit of about $2,887 for three people, and net income of $1,258 is below the $2,221 poverty line, so the household qualifies for $407 a month." -us,scenario_012,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Using a $224 standard deduction instead of $209, the model got about $412 a month with no shelter deduction. It then replaced that with an unexplained $601 a month. The correct net income is $1,258 with no shelter deduction, giving $785 − $378 = $407." -us,scenario_012,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model first found no shelter deduction and about $406 a month (using a $204 standard deduction instead of $209). It then 're-estimated with shelter' to $514 a month. No excess shelter deduction applies because the $450 rent is less than half of the $1,258 net income and the energy subsidy gives no utility allowance." -us,scenario_012,snap,claude-opus-5,llm_error,other,False,"The model reported $378 a month, which is the 30% contribution from the $1,258 net income, as the benefit itself. It never subtracted that from the $785 maximum, which leaves $407." -us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed the energy subsidy entitles the household to a standard utility allowance (about $395) and took a $216 excess shelter deduction. Under the 2026 rules, energy assistance gives that allowance only to households with an elderly or disabled member. Shelter cost is therefore just the $450 rent, below half of the $1,258 net income, so there is no deduction." -us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a fourth SNAP member, using four-person amounts ($973 maximum) and FY2025 figures. It also added an energy-subsidy utility allowance of about $414. It then submitted $5,136, which does not match its own $7,968 calculation. The correct unit is three people with no shelter deduction, which gives $407 a month." -us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model treated the pregnant spouse as adding a fourth member (about $994 maximum allotment) and added a utility allowance because of the energy subsidy. SNAP does not count an unborn child, and this household gets no utility allowance, so the correct figure is the three-person $785 maximum minus $378." -us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"The model correctly found net income of $1,258 and no shelter deduction, but rounded the result to about $408 a month. SNAP rounds 30% of net income up to $378, which gives $785 − $378 = $407, or $4,884 a year." -us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $211 standard deduction and a $786 maximum allotment. The FY2026 three-person amounts are $209 and $785, which give net income of $1,258, a $378 contribution and a $407 benefit, not $409." -us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added a $500 standard utility allowance to the rent, producing a $316.67 excess shelter deduction, and used a $200 standard deduction. The household has no utility allowance, so shelter cost is only $450, below half of the $1,258 net income, and the benefit is $407 a month." -us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used outdated FY2025 amounts: a $204 standard deduction and a $768 maximum allotment. With the FY2026 amounts of $209 and $785, net income is $1,258 and the benefit is $407 a month." -us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a member and used a four-person household. SNAP counts only the three people present, so the $785 three-person maximum applies and the benefit is $407 a month." -us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model said the energy subsidy triggered a standard utility allowance and applied an excess shelter deduction. For a household with no elderly or disabled member, energy assistance does not give a utility allowance in 2026. The $450 rent alone is below half of the $1,258 net income, so no shelter deduction applies." -us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave $446 a month with no derivation. That is $39 above the correct $407, which matches an unsupported deduction cutting net income by about $130. No shelter deduction applies, so net income stays at $1,258 and the contribution at $378." -us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction that included a standard utility allowance. The household pays no utilities, and the energy subsidy does not give a utility allowance to a non-elderly, non-disabled household. Shelter cost is therefore just $450 and no excess shelter deduction applies." -us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied a shelter deduction that lowered net income enough to produce $457 a month. Only the $450 rent counts as a shelter cost, which is below half of the $1,258 net income, so the benefit is $785 − $378 = $407." -us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"The model gave $1,450 a year with no derivation, which implies a contribution of about $664 a month. That is consistent with using roughly gross income instead of net income after the $209 standard deduction and the 20% earned-income deduction. The correct net income is $1,258, giving a $378 contribution and a $407 monthly benefit." -us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction that does not exist here. With only the $450 rent counted, half of the $1,258 net income ($629) is higher, so the benefit stays at $407 a month." -us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model included shelter costs as a deduction, implying a utility allowance from the energy subsidy. That allowance is not available to this household, and the $450 rent alone produces no excess shelter deduction, so the benefit is $407 a month." -us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $200 monthly standard deduction ($2,400 a year) and the FY2025 $768 maximum ($9,216 a year). The FY2026 amounts are $209 ($2,508 a year) and $785 ($9,420 a year), which give a $407 monthly benefit." -us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added a fourth member for the pregnancy, which led it to use a $975 maximum and a $238 standard deduction. It also applied a $360 utility allowance because of the energy subsidy. SNAP counts three people here and gives no utility allowance, so the benefit is $785 − $378." -us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model added a utility allowance based on the energy subsidy to the $450 rent, creating an excess shelter deduction and $1,044 net income. It also used a $204 standard deduction instead of $209. Without the utility allowance there is no shelter deduction, so net income is $1,258 and the benefit is $407." -us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model counted housing and utility costs toward a shelter deduction. The household has no utility allowance, and the $450 rent is below half of the $1,258 net income, so no deduction applies and the benefit is $407 a month." -us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned 0, saying more eligibility inputs were needed. The facts given are enough: a three-person household with $1,833 monthly earnings and $2 in assets passes the gross and net income tests and receives $407 a month." -us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed the energy subsidy qualifies the household for the heating and cooling utility allowance. Under the 2026 rules, energy assistance gives that allowance only to households with an elderly or disabled member, so no excess shelter deduction applies and net income stays at $1,258." -us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"The model correctly used no utility allowance, but got $408 a month. SNAP rounds 30% of the $1,258 net income up to $378, which leaves a $407 benefit, or $4,884 a year." -us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied a heating and cooling utility deduction on top of the rent. This household does not qualify for that allowance, so shelter cost is only $450, below half of net income, and the benefit is $407 a month, not $478." -us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"The model reported $408 a month instead of $407. SNAP rounds net income to $1,258 and the 30% contribution up to $378, giving $785 − $378 = $407 and $4,884 a year." -us,scenario_012,snap,gpt-6-astra,llm_error,other,False,"The model correctly ruled out the energy-subsidy utility allowance and the shelter deduction, but worked on annual totals without SNAP's monthly rounding. Monthly net income rounds to $1,258, its 30% contribution rounds up to $378, and the benefit is $407 a month, or $4,884, not $4,892.40." -us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $406.20 monthly benefit implies a contribution of $378.80 and net income of $1,262.67, which means it used the FY2025 $204 standard deduction instead of the FY2026 $209. It also skipped rounding. The correct figures are net income $1,258, contribution $378 and benefit $407." -us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model included a shelter deduction. With no utility allowance, the $450 rent is below half of the $1,258 net income, so no shelter deduction applies and the benefit is $407 a month, not $504." -us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used the correct $785 maximum and $209 standard deduction, but added a $433 heating utility allowance because of the energy subsidy. Energy assistance does not give a utility allowance to this non-elderly, non-disabled household in 2026, so no shelter deduction applies and the benefit is $407 a month." -us,scenario_012,snap,grok-4.3,llm_error,other,False,"The model gave $3,600 ($300 a month) with no calculation. The correct result is $785 minus a $378 contribution on $1,258 net income, which is $407 a month. Its figure understates the benefit by about $107 a month." -us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used a $198 standard deduction and a $766 maximum allotment instead of the FY2026 $209 and $785, getting $385 a month. It also multiplied $385 by 12 wrongly, reporting $4,625 instead of $4,620." -us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model gave the household a standard utility allowance triggered by the energy subsidy and an excess shelter deduction. In 2026 that allowance requires an elderly or disabled member, so shelter cost is only $450, below half of net income, and the benefit is $407 a month." -us,scenario_012,snap,grok-4.7,llm_error,other,False,"The deductions, the $785 maximum and the lack of a shelter deduction were all right, but the model calculated on annual totals without SNAP rounding. Net income rounds to $1,258 a month and the 30% contribution rounds up to $378, giving $407 a month and $4,884, not $4,892.40." -us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model subtracted the $210 monthly standard deduction from annual income instead of the annual $2,508 ($209 × 12). This overstated net income by about $2,300 a year and inflated the contribution. It also counted the $411 energy subsidy as a shelter cost." -us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value and no explanation, so there was no answer to score." -us,scenario_012,snap,kimi-k3,llm_error,other,False,"The derivation (net income $1,257.67, no shelter deduction, $785 maximum) is right, but the model skipped SNAP's rounding. Net income rounds to $1,258, the 30% contribution rounds up to $378, and the benefit is $407 a month, not $407.70." -us,scenario_012,snap,minimax-m3,llm_error,other,False,"The model returned 0 without doing any calculation. Gross income of $1,833 passes the three-person gross limit and net income of $1,258 passes the net test, so the benefit is $785 − $378 = $407 a month." -us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model applied a standard utility allowance of about $470 funded by energy assistance (LIHEAP), creating a $290 excess shelter deduction, and used a $204 standard deduction. Energy assistance gives no utility allowance to this non-elderly, non-disabled household, so net income is $1,258 and the benefit is $407 a month." -us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted the $411 energy subsidy as income and used a $198 standard deduction and a $768 maximum allotment. It then switched to an unexplained $426 a month. Energy assistance is excluded from SNAP income, and the FY2026 amounts are $209 and $785, which give $407 a month." -us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used the full $22,000 gross earnings as net income for the net income test. It skipped the 20% earned-income deduction and the $209 standard deduction, which bring net income down to $1,258 a month, well below the $2,221 poverty line." +us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It wrongly gave the household a roughly $400 SUA because of the energy subsidy, which created an excess shelter deduction. Its final $378/month is exactly the household's expected contribution (30% of $1,258, rounded up), not the allotment. The allotment is $785 − $378 = $407." +us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"Its deductions, household size, and maximum allotment were all correct. It skipped SNAP's rounding rule: 30% of net income ($377.40) rounds up to $378. That makes the benefit $407/month, not $407.70, so the annual total is $4,884 rather than $4,892." +us,scenario_012,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It left out the 20% earned income deduction entirely. It also used a $256 standard deduction and an outdated $694 maximum allotment instead of $209 and $785. It then treated the energy subsidy as an adjustment to net income, giving $84/month instead of $407." +us,scenario_012,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It counted four people and declared the household ineligible. Gross income of $1,833/month is far below 130% of the $2,220.83 monthly poverty guideline for three people, and net income of $1,258 passes the net test. The household gets $407/month." +us,scenario_012,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached about $412/month using a $224 standard deduction instead of $209. It then dropped that result and moved to $601/month with no deduction to support it. The correct figure is net income $1,258, contribution $378, and benefit $407." +us,scenario_012,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It correctly found that no shelter deduction applies and got about $406/month, using a $204 standard deduction instead of $209. It then inflated the result to $514/month by reintroducing a shelter deduction that does not exist." +us,scenario_012,snap,claude-opus-5,llm_error,other,False,"Its $378/month is exactly the expected household contribution (30% of $1,258 net income, rounded up). It reported that contribution as the benefit instead of subtracting it from the $785 maximum, which gives $407." +us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It treated the energy subsidy as qualifying the household for a roughly $395 SUA and took a $216 excess shelter deduction. Under the 2026 rules, energy assistance confers the SUA only on households with an elderly or disabled member. With rent alone ($450) below half of net income ($629), there is no shelter deduction." +us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It counted the unborn child as a fourth SNAP member, which PolicyEngine does not do. That gave it the four-person $973 maximum allotment. It also added an SUA because of the energy subsidy, and it submitted $5,136 even though its own reasoning reached $7,968." +us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used a household size of 4 for the pregnancy and a roughly $994 maximum allotment, when the SNAP unit is three people with a $785 maximum. It also added a utility allowance tied to the energy subsidy, although no shelter deduction applies." +us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"Its structure was correct ($1,258 net income, no shelter deduction, $785 maximum). It did not round the 30% contribution up to $378, so it got about $408/month and $4,890 instead of $407/month and $4,884." +us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $211 standard deduction and a $786 maximum allotment instead of the FY2026 three-person values of $209 and $785. It also left net income unrounded, giving $409/month instead of $407." +us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It added a $500 SUA to rent and took a $316.67 excess shelter deduction. The household has no utility expenses, and the energy subsidy does not confer the SUA on a household with no elderly or disabled member. It also used a $200 standard deduction instead of $209." +us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 values, a $768 maximum allotment and a $204 standard deduction, instead of FY2026's $785 and $209. That produced $389.20/month instead of $407." +us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"It counted the unborn child as a member, making the SNAP household four people and inflating the maximum allotment. The unit has three members with a $785 maximum, which gives $407/month." +us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied a standard utility allowance that it said the energy subsidy triggered, which created an excess shelter deduction and a benefit of about $482/month. Under 2026 rules, energy assistance does not confer the SUA on a household with no elderly or disabled member. Rent of $450 is below half of the $1,258 net income." +us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $446/month means it counted about $130 less net income than the correct $1,258, which matches a shelter deduction that does not apply. The correct path is $1,833.33 − $209 − $366.67 = $1,258 net income, a $378 contribution, and a $407 benefit." +us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It took an excess shelter deduction based on rent plus the standard utility allowance. The energy subsidy does not qualify this non-elderly, non-disabled household for the SUA. With rent alone, shelter costs are below half of net income and no deduction applies." +us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied a shelter deduction that cut net income below $1,258 and produced $457/month. Rent of $450 is below half of net income ($629), and no utility allowance applies. The benefit is $407/month." +us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. $1,450/year is about $121/month, which is consistent with skipping the 20% earned income deduction or using a far smaller maximum allotment. The correct calculation is $1,258 net income and $785 − $378 = $407/month." +us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It explicitly took an excess shelter deduction, which produced $453/month. Shelter costs are rent alone ($450), below half of net income ($629), so there is no deduction and the benefit is $407/month." +us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It counted shelter costs as a deduction, which lowered net income and raised the benefit to about $472/month. With no utility allowance and rent below half of net income, no excess shelter deduction applies." +us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $200/month standard deduction and the FY2025 $768 maximum allotment ($9,216/year) instead of FY2026's $209 and $785. It also skipped monthly rounding, giving $4,656 instead of $4,884." +us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"It made the household four people by adding the pregnancy, then used a $975 maximum and a $238 standard deduction. It also gave the household an SUA because of the energy subsidy. The unit has three people, the maximum is $785, and no shelter deduction applies." +us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added a utility allowance based on the energy subsidy and took a shelter deduction, cutting net income to about $1,044. It also used about $780 and $204 instead of the $785 maximum and $209 standard deduction. Net income is $1,258 with no shelter deduction." +us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It cited housing and utility costs. $497/month is consistent with an SUA-based excess shelter deduction that does not apply, because the household has no utility expenses and gets no SUA from the energy subsidy. The benefit is $407/month." +us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned 0, saying SNAP eligibility inputs were missing. All three members meet the work and immigration requirements, assets are $2, and income passes the gross and net tests. The household is eligible for $407/month." +us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It said the energy subsidy qualifies the unit for the heating/cooling utility allowance and took an excess shelter deduction. Under 2026 rules, energy assistance confers the SUA only on households with an elderly or disabled member, so net income stays at $1,258 and the benefit is $407/month." +us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"It correctly applied no utility allowance and no shelter deduction, but got $408/month. It did not round the 30% contribution ($377.40) up to $378, which gives $407/month." +us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied a heating/cooling utility deduction, which produced $478/month. The energy subsidy does not confer the SUA on this household, and rent alone is below half of net income, so there is no shelter deduction." +us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"It got $408/month instead of $407 because it did not round 30% of the $1,258 net income ($377.40) up to $378 before subtracting it from the $785 maximum." +us,scenario_012,snap,gpt-6-astra,llm_error,period_annualization,False,"It correctly rejected the SUA but computed annually: $9,420 − 0.3 × $15,092. This skipped the monthly rounding steps. Net income rounds to $1,258, the contribution rounds up to $378, and the allotment is a whole-dollar $407/month." +us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"$406.20/month implies net income of $1,262.67, which comes from the FY2025 $204 standard deduction instead of FY2026's $209. It also skipped rounding net income and rounding the contribution up." +us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It included a shelter deduction, which produced $504/month. Rent of $450 is below half of the $1,258 net income, and no utility allowance applies, so the deduction is zero and the benefit is $407/month." +us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It added a $433 heating utility allowance that it said the energy subsidy triggered. Under the 2026 rules, energy assistance does not confer the SUA on a household with no elderly or disabled member, so no excess shelter deduction applies." +us,scenario_012,snap,grok-4.3,llm_error,other,False,"It gave no derivation. $300/month falls well short of the correct $785 − 0.3 × $1,258 (rounded up to $378) = $407/month, which shows it did not apply the FY2026 maximum allotment to the correctly computed net income." +us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"It used outdated parameters, a $198 standard deduction and a $766 maximum allotment, instead of FY2026's $209 and $785. That gave $385/month instead of $407." +us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It took an excess shelter deduction that included a LIHEAP-triggered SUA. Under 2026 rules, LIHEAP confers the SUA only on households with an elderly or disabled member, so this household gets no shelter deduction and $407/month." +us,scenario_012,snap,grok-4.7,llm_error,period_annualization,False,"It computed the benefit on annual totals ($9,420 − 0.3 × $15,092) and skipped the monthly rounding steps. Net income rounds to $1,258, the contribution rounds up to $378, and the benefit is $407/month, or $4,884." +us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"It subtracted the monthly standard deduction ($210) from annual income once instead of $209 × 12 = $2,508. This overstated annual net income by about $2,300, and it then used a projected $9,422 maximum allotment." +us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there was no answer to score against the $4,884 reference." +us,scenario_012,snap,kimi-k3,llm_error,other,False,"It correctly got $1,257.67 net income with no shelter deduction, but did not round net income to $1,258 or round the 30% contribution up to $378. That gave $407.70/month instead of $407." +us,scenario_012,snap,minimax-m3,llm_error,categorical_eligibility,False,"It returned 0 without doing the calculation. The household passes the gross and net income tests with $1,258 net income, which yields $785 − $378 = $407/month." +us,scenario_012,snap,muse-wonder,llm_error,thresholds_rates,False,"It used the FY2025 three-person maximum allotment of $768 instead of FY2026's $785, so it got about $391/month instead of $407." +us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added a roughly $470 LIHEAP-funded SUA to rent and took about a $290 excess shelter deduction. Under 2026 rules, LIHEAP does not confer the SUA on a household with no elderly or disabled member. It also used a $204 standard deduction instead of $209." +us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the $411 energy subsidy as unearned income, but that subsidy is excluded from SNAP income. It also used an outdated $198 standard deduction and $768 maximum, then gave an unsupported $426/month." +us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It set net income equal to gross earnings, leaving out the 20% earned income deduction and the $209 standard deduction. That made the household fail the net income test, when its net income is actually $1,258/month, well under the poverty guideline." us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated pregnancy and income below a pregnancy-coverage threshold as sufficient for CHIP. It omitted the prior Medicaid screen: the 18-year-old spouse qualifies for Medicaid under the OLDER_CHILD category, which makes her ineligible for CHIP." us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that pregnancy creates CHIP eligibility regardless of age once income requirements are met. It failed to apply the rule excluding anyone already Medicaid-eligible; this spouse is Medicaid-eligible as an OLDER_CHILD. us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified low-income pregnancy coverage as CHIP-funded coverage without first testing Medicaid eligibility. The spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP, irrespective of household size adjustments for the unborn child or pregnancy income limits." @@ -1004,91 +1019,93 @@ us,scenario_013,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,t us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model asserted that income was ""near SSI/FPL-based aged-disabled standards"" but never compared it to a limit. The head's countable income is about $30,5xx: $23,736 of Social Security plus the $6,720 pension and $88 of dividends and capital gains, less the $20 general disregard. That is about 1.9x FPL, well above the 100% FPL aged/disabled income standard and the SSI benefit rate. SSI is 0, so no SSI-linked pathway applies, and at age 80 the head is outside the MAGI adult group." us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model said Social Security is ""mostly excluded via SSI methodology"" and cut it to $10,196 countable. SSI methodology counts Social Security in full as unearned income, minus only the $20 general disregard, so countable income is about $30,5xx, or 1.9x FPL, above the aged/disabled standard. The model also applied the ALTCS 300%-of-SSI limit, which requires an institutional level-of-care need this household does not have. It then deducted health premiums and OTC costs as a spend-down, which does not create eligibility here, and it ignored its own finding that the $58,700 in assets exceeds the $2,000 limit." us,scenario_013,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model tested AGI of $6,808 against 138% FPL. MAGI adds back the $23,736 of nontaxable Social Security, which puts income at about $30,544, or 1.91x FPL. The 138% FPL adult expansion group also covers only ages 19-64, so it cannot apply to an 80-year-old. The aged/disabled pathway counts total income including Social Security, and that income is far above its limit." -us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied only the regular elderly/disabled rules: the 100% FPL net income test and the asset limit. It never checked Arizona's BBCE. From March 2026, gross income of about $2,539/month is under the 200% FPL limit ($2,608), so both tests count as met. As a one-person household it then gets the $24 minimum for 10 months." -us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Concluded the household is ineligible because net income (~$2,322) exceeds the 100% FPL net limit. It missed that Arizona's BBCE covers gross income up to 200% FPL ($2,608/month) from March 2026 and waives the net income test. A categorically eligible one-person household gets the $24 minimum benefit, which is $240 for March–December." -us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross income test and a $2,500 asset limit. An elderly/disabled household is exempt from the gross test. More importantly, Arizona's BBCE at 200% FPL from March 2026 makes this household categorically eligible and waives the asset test, so it gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Denied benefits because the aged/disabled income and asset tests were not met. It never considered Arizona's BBCE: from March 2026, gross income of $2,539/month is under 200% FPL ($2,608), which waives those tests. The result is the $24 one-person minimum benefit for 10 months, or $240." -us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Ran only the regular net income test (net ~$2,265 vs ~$1,255) and stopped at ineligible. It missed Arizona's March 2026 BBCE limit of 200% FPL ($2,608/month gross), which this household meets. That waives the net income test and entitles a one-person household to the $24 minimum benefit for March–December." -us,scenario_013,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Treated the $58,700 bank balance as disqualifying under the ~$4,500 elderly/disabled resource limit. Arizona's BBCE removes the asset test for households with gross income up to 200% FPL from March 2026, and this household's $2,539/month qualifies. It then receives the $24 one-person minimum for 10 months." -us,scenario_013,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Denied SNAP over the asset limit and never applied Arizona's BBCE. From March 2026, BBCE covers gross income up to 200% FPL ($2,608/month), so the asset and net income tests count as met. The household then gets the $24 one-person minimum benefit, which is $240 for the year." -us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Concluded zero because net income (~$2,330) exceeds 100% FPL. It missed that Arizona's BBCE (200% FPL gross limit from March 2026) waives the net income test. A categorically eligible one-person household whose formula benefit is below zero still receives the $24 minimum, giving $240 for 10 months." -us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Applied the regular net income test and the $4,500 elderly/disabled asset limit, and treated the $298 annual premium as a monthly expense. It never applied Arizona's BBCE: gross income of $2,539/month is under the 200% FPL limit from March 2026, which waives both tests. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Correctly tested against a 200% FPL gross limit, but computed it as ~$2,510/month from the stale 2024 poverty guideline and added the $72 capital gains into gross income. The correct limit is $2,608.33 from March 2026 ($2,660 from October), and gross income of $2,539 falls under it. That makes the household categorically eligible for the $24 minimum for 10 months." -us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Took $298 minus 30% of net income going negative to mean a zero benefit. It missed that the household is categorically eligible through Arizona's BBCE (gross under 200% FPL from March 2026). Eligible one-person households receive the $24 minimum even when the formula is below zero, so the answer is $240 for March–December." -us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Denied SNAP over the elderly/disabled asset limit. From March 2026, Arizona's BBCE covers gross income up to 200% FPL ($2,608/month), which this household meets, and that waives the asset test. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Applied the $4,250 elderly/disabled asset limit as decisive. Arizona's BBCE at 200% FPL from March 2026 makes the household categorically eligible, which means the asset limit does not apply. The household then receives the $24 one-person minimum benefit for 10 months, or $240." -us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Relied on the regular net income and resource tests. It missed that from March 2026, Arizona's BBCE treats both tests as met for gross income under 200% FPL ($2,608/month vs. the household's $2,539). That entitles the one-person household to the $24 minimum for 10 months." -us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Concluded ineligibility from the net income limit and the asset limit. Both tests are waived under Arizona's BBCE, which covers gross income up to 200% FPL from March 2026. The household then receives the $24 one-person minimum benefit for March–December, or $240." -us,scenario_013,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used Arizona's old 185% FPL gross limit (~$2,321/month). From March 2026, Arizona's BBCE limit is 200% FPL ($2,608.33/month), and the household's $2,539 gross income meets it. That waives the net income test and yields the $24 one-person minimum for 10 months." -us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Denied SNAP under the regular resource limit and net income test. It never applied Arizona's BBCE, which from March 2026 treats both tests as met for gross income up to 200% FPL. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Applied only the elderly net income limit. Arizona's BBCE (200% FPL gross from March 2026) makes this household categorically eligible and waives that limit. A categorically eligible one-person household receives the $24 minimum benefit for 10 months, or $240." -us,scenario_013,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"Tested BBCE against Arizona's old 185% FPL limit, concluded the household fell outside it, and then applied the $4,250 asset limit. From March 2026, Arizona's BBCE limit is 200% FPL ($2,608/month), and $2,539 gross income qualifies. That waives the asset test and yields the $24 minimum for 10 months." -us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Denied SNAP on income and asset thresholds without applying Arizona's BBCE. From March 2026, gross income under 200% FPL ($2,608/month) makes the household categorically eligible. It then receives the $24 one-person minimum for March–December." -us,scenario_013,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"Asserted that gross income exceeds Arizona's eligibility threshold. From March 2026, Arizona's BBCE gross limit is 200% FPL ($2,608.33/month), above the household's $2,539.33. The household is categorically eligible for the $24 one-person minimum for 10 months." -us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Treated a formula result at or below zero as a zero benefit. Under Arizona's BBCE (200% FPL from March 2026), the household is categorically eligible and the net income test is waived. Eligible one-person households receive the $24 minimum benefit, which comes to $240 for 10 months." -us,scenario_013,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"Applied Arizona's BBCE at the old 185% FPL limit and then the elderly asset limit. From March 2026, the BBCE limit is 200% FPL ($2,608/month), and $2,539 gross income meets it. That waives the asset test and entitles the household to the $24 one-person minimum for 10 months." -us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,"Compared gross income with the 100% FPL net income limit and stopped. It missed Arizona's BBCE: from March 2026, gross income under 200% FPL makes the household categorically eligible, and the household then gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"Computed net income of ~$27,012/year (including $72 of capital gains in income) and denied SNAP for exceeding the gross and net limits. An elderly/disabled household is exempt from the gross test, and Arizona's BBCE (200% FPL from March 2026) waives the net test. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Called the income too high without checking Arizona's BBCE. From March 2026, gross income of $2,539/month is under the 200% FPL limit ($2,608), so the household is categorically eligible and receives the $24 one-person minimum for 10 months." -us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to zero without any eligibility analysis. The correct path: monthly gross income of $2,539.33 falls under Arizona's 200% FPL BBCE limit from March 2026, which waives the income and asset tests. That yields the $24 one-person minimum for 10 months, or $240." -us,scenario_013,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Denied SNAP because $58,700 exceeds the elderly/disabled resource limit. From March 2026, Arizona's BBCE covers gross income up to 200% FPL and waives the resource test. The household receives the $24 one-person minimum for March–December." -us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Took the formula allotment (below zero) as the final answer. As a household categorically eligible through Arizona's 200% FPL BBCE from March 2026, a one-person household receives the $24 minimum benefit instead of zero, or $240 for 10 months." -us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Concluded that income after deductions produces no allotment. It missed that BBCE eligibility (gross under Arizona's 200% FPL limit from March 2026) entitles a one-person household to the $24 minimum benefit even when the formula is negative, or $240 for 10 months." -us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,Applied the regular 100% FPL net income test. Arizona's BBCE (200% FPL gross limit from March 2026) treats that test as met for this household. The one-person household then receives the $24 minimum for 10 months. -us,scenario_013,snap,gpt-6-astra,llm_error,state_local_rule,False,"Asserted that income exceeds Arizona's categorical eligibility limit, which reflects the pre-2026 185% FPL threshold. From March 2026 the limit is 200% FPL ($2,608.33/month), and the household's $2,539.33 qualifies. That waives the asset test and yields the $24 one-person minimum for 10 months." -us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Equated a non-positive formula allotment with zero SNAP. The household is categorically eligible under Arizona's 200% FPL BBCE from March 2026, so as a one-person household it receives the $24 minimum benefit for 10 months, or $240." -us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Concluded that income after deductions produces no allotment. It never applied Arizona's March 2026 BBCE expansion to 200% FPL, which makes the household eligible. It also missed the $24 one-person minimum benefit that replaces a negative formula result, for $240 over 10 months." -us,scenario_013,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"Asserted that income exceeds Arizona's expanded categorical eligibility limit, using the old 185% FPL cap. From March 2026 the limit is 200% FPL ($2,608.33/month), above the household's $2,539.33. That waives the resource test and gives the $24 one-person minimum for 10 months." -us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"Denied SNAP on regular income and asset limits. Arizona's BBCE at 200% FPL from March 2026 waives both tests for this household's $2,539/month gross income. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit after standard and medical deductions. That test is treated as met under Arizona's BBCE, which covers gross income up to 200% FPL from March 2026. The one-person household then receives the $24 minimum for March–December." -us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"Denied SNAP on the net income limit and the elderly/disabled resource limit. Both are waived under Arizona's BBCE: gross income of $2,539/month is under 200% FPL ($2,608) from March 2026. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the $298 maximum, but treated that as a zero benefit. Through Arizona's 200% FPL BBCE from March 2026, the household is categorically eligible. A one-person household then receives the $24 minimum benefit, or $240 for 10 months." -us,scenario_013,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Applied the $4,250 elderly/disabled resource limit as disqualifying. Arizona's BBCE waives the resource test for gross income up to 200% FPL from March 2026, and this household qualifies. It receives the $24 one-person minimum for 10 months." -us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"Applied the regular 100% FPL net income test (net ~$2,310 vs ~$1,255) and denied benefits. Arizona's BBCE (200% FPL gross from March 2026) waives that test. The household receives the $24 one-person minimum benefit for 10 months, or $240." -us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Denied SNAP on the federal elderly/disabled resource limit and the 100% FPL net income test. Arizona's BBCE covers gross income up to 200% FPL from March 2026, and that treats both tests as met. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"Applied the one-person net income limit to net income of ~$2,305 and denied benefits. Under Arizona's BBCE (gross under 200% FPL from March 2026), the net income test counts as met. The household receives the $24 one-person minimum for March–December." -us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"Treated income and assets as exceeding elderly SNAP thresholds. It missed Arizona's March 2026 BBCE expansion to 200% FPL, which this $2,539/month gross income meets and which waives both tests. The household gets the $24 one-person minimum for 10 months." -us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly exempted the elderly/disabled household from the gross test, then denied benefits on the 100% FPL net income test. Arizona's BBCE at 200% FPL from March 2026 waives that net test. The household then receives the $24 one-person minimum for 10 months." -us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test, but elderly/disabled households are exempt from it. It also missed Arizona's BBCE at 200% FPL from March 2026, which makes this household categorically eligible for the $24 one-person minimum for 10 months." -us,scenario_013,snap,qwen3.8-max,llm_error,age_disability,False,"Invented a rule that households of members aged 60+ are categorically ineligible for SNAP. In fact, elderly households get favorable treatment, and under Arizona's 200% FPL BBCE from March 2026 this household qualifies. It receives the $24 one-person minimum for 10 months." +us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It applied the ordinary 100% FPL net income test and a $4,500 elderly/disabled asset limit, and never considered Arizona's BBCE. From March 2026, BBCE at 200% FPL ($2,608.33/month) deems both tests met for this $2,539.33 gross income. It therefore missed the $24 monthly minimum for one-person households in March–December ($240 a year)." +us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It ruled the household out with the 100% FPL net income test and never applied Arizona's 200% FPL BBCE, which starts in March 2026. BBCE makes this household (gross $2,539.33 < $2,608.33) categorically eligible, so it gets the $24 one-person minimum allotment for 10 months even though the benefit formula comes out negative." +us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a 130% FPL gross income limit, which elderly/disabled households are exempt from, and a $2,500 asset limit. It missed that Arizona's 200% FPL BBCE (from March 2026) deems both the income and asset tests met for $2,539.33 gross monthly income. That leaves the household with the $24 monthly minimum for March–December." +us,scenario_013,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It applied the ordinary aged/disabled income and asset tests and ignored Arizona's broad-based categorical eligibility. At 200% FPL from March 2026, BBCE waives both tests for this household. The one-person $24 minimum allotment then applies for ten months, totaling $240." +us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"It stopped at the 100% FPL net income test (net about $2,265) and never checked Arizona's BBCE. BBCE's 200% FPL gross limit ($2,608.33 from March 2026) treats the net income test as met. The household should have gotten the $24 one-person minimum for March–December." +us,scenario_013,snap,claude-opus-4.8,llm_error,asset_resource,False,"It disqualified the household for exceeding the $4,500 elderly/disabled resource limit. It missed that Arizona's BBCE (200% FPL gross limit from March 2026, met at $2,539.33/month) removes the asset test entirely, which leaves a $24 monthly minimum allotment for ten months." +us,scenario_013,snap,claude-opus-5,llm_error,asset_resource,False,"It treated the $58,700 in bank assets as disqualifying under the federal resource limit. Arizona's BBCE at 200% FPL, effective March 2026, makes this household categorically eligible and waives the asset test. The household then gets the $24 one-person minimum for March–December." +us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit and stopped there. It never applied Arizona's 200% FPL BBCE (from March 2026), under which the household's $2,539.33 gross income makes it categorically eligible. A categorically eligible one-person household gets the $24 minimum allotment even when the formula comes out negative." +us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It applied the ordinary net income limit and a $4,500 resource limit, and never considered Arizona's broad-based categorical eligibility. From March 2026, BBCE at 200% FPL ($2,608.33/month) deems both tests met for $2,539.33 gross income. That entitles the household to the $24 one-person minimum for ten months." +us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It recognized a 200% FPL gross limit but put it at about $2,510/month; the correct figure is $2,608.33 ($2,660 from October). It also counted the $72 capital gains as income. With the right threshold, gross income of $2,539.33 qualifies for BBCE, which waives the asset test and yields the $24 minimum for March–December." +us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It ran the $298 minus 30%-of-net-income formula and stopped at zero. It missed that Arizona's 200% FPL BBCE makes the household categorically eligible from March 2026, and that eligible one-person households get the $24 minimum allotment, for $240 a year." +us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It disqualified the household on the elderly/disabled asset limit. Arizona's BBCE (200% FPL gross limit from March 2026, met at $2,539.33/month) eliminates the asset test, so the household gets the $24 one-person minimum for ten months." +us,scenario_013,snap,deepseek-v4-pro,llm_error,asset_resource,False,"It applied the $4,250 elderly/disabled resource limit as decisive. From March 2026, Arizona's 200% FPL BBCE deems the asset test met for this household (gross $2,539.33 < $2,608.33), which produces the $24 monthly minimum for March–December." +us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It applied the ordinary net income limit and the elderly/disabled resource limit. It never considered Arizona's 200% FPL BBCE from March 2026, which treats both tests as met, and so it missed the $24 one-person minimum allotment for ten months." +us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It ruled the household ineligible on the ordinary net income and asset limits, and never applied Arizona's BBCE at 200% FPL (effective March 2026). BBCE makes the household categorically eligible, and the $24 minimum allotment then applies for March–December." +us,scenario_013,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 185% FPL gross limit of about $2,321/month. From March 2026, Arizona's BBCE limit is 200% FPL ($2,608.33/month), which the household's $2,539.33 gross income meets. Categorical eligibility then gives the $24 one-person minimum for ten months, even though the formula comes out negative." +us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It applied the ordinary resource limit and net income test. It missed Arizona's 200% FPL BBCE from March 2026, which waives both for this household and yields the $24 one-person minimum allotment for March–December." +us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It applied the elderly net income limit and stopped. From March 2026, Arizona's BBCE at 200% FPL ($2,608.33/month) treats the net income test as met for $2,539.33 gross income, so the household gets the $24 minimum for ten months." +us,scenario_013,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used Arizona's old 185% FPL BBCE limit and applied the $4,250 asset test. From March 2026, Arizona's BBCE limit is 200% FPL ($2,608.33/month), which the household's $2,539.33 gross income meets. The asset test is then waived and the $24 one-person minimum applies for March–December." +us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It disqualified the household on ordinary income and asset limits. It missed that Arizona's 200% FPL BBCE (from March 2026) deems both met for $2,539.33 gross monthly income, which yields the $24 minimum allotment for ten months." +us,scenario_013,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It claimed gross income exceeds Arizona's SNAP threshold, which matches the old 185% FPL BBCE limit. From March 2026, the limit is 200% FPL ($2,608.33/month), which $2,539.33 gross income satisfies. That makes the household categorically eligible for the $24 minimum for March–December." +us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It applied the net income limit and let the benefit formula zero out the benefit. It missed that Arizona's 200% FPL BBCE makes the household categorically eligible from March 2026, and that eligible one-person households receive the $24 minimum allotment." +us,scenario_013,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used Arizona's old 185% FPL BBCE limit and then applied the federal elderly asset test. Arizona raised BBCE to 200% FPL ($2,608.33/month) in March 2026, so gross income of $2,539.33 qualifies, the asset test is waived, and the household gets $24 a month for ten months." +us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,"It applied only the 100% FPL net income limit for elderly households. It never applied Arizona's 200% FPL BBCE from March 2026, which deems income eligibility met at $2,539.33 gross monthly income, and so it missed the $24 one-person minimum for March–December." +us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"It compared net income to the ordinary gross and net limits. It ignored Arizona's 200% FPL BBCE (from March 2026), which makes the household categorically eligible despite failing those tests. The benefit is therefore the $24 one-person minimum for ten months, not zero." +us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It judged Social Security plus pension income too high under the ordinary rules. It missed that Arizona's 200% FPL BBCE ($2,608.33/month from March 2026) covers this $2,539.33 gross income and guarantees the $24 one-person minimum allotment." +us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It gave no substantive derivation and assumed no positive SNAP benefit was possible. In fact, Arizona's 200% FPL BBCE from March 2026 makes this household categorically eligible, which yields the $24 one-person minimum for ten months ($240)." +us,scenario_013,snap,gpt-5.5,llm_error,asset_resource,False,"It disqualified the household for exceeding the elderly/disabled resource limit. Arizona's BBCE at 200% FPL, effective March 2026 and met at $2,539.33/month, waives the asset test, so the household gets the $24 minimum for March–December." +us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It took the formula's zero allotment as final. It missed that Arizona's 200% FPL BBCE makes the household categorically eligible from March 2026, and that eligible one-person households get the $24 minimum benefit even when the formula comes out negative." +us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It concluded that income left no positive allotment after deductions. It never applied Arizona's 200% FPL BBCE (from March 2026), under which the household is categorically eligible and receives the $24 one-person minimum for ten months." +us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It applied the ordinary net income limit. Arizona's 200% FPL BBCE, effective March 2026, deems that test met for gross income of $2,539.33 (under $2,608.33), which yields the $24 minimum allotment for March–December." +us,scenario_013,snap,gpt-6-astra,llm_error,thresholds_rates,False,"It found income above Arizona's categorical eligibility limit, which reflects the old 185% FPL threshold. From March 2026 the limit is 200% FPL ($2,608.33/month, $2,660 from October), which the household's $2,539.33 gross income meets. The resource limit is then waived and the household gets $24 a month for ten months." +us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It stopped at the formula result of zero. It missed that Arizona's 200% FPL BBCE makes the household categorically eligible from March 2026, so the $24 one-person minimum allotment applies for ten months." +us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It treated the zero computed allotment as the answer. It never applied Arizona's 200% FPL BBCE from March 2026, which makes the household eligible and entitles it to the $24 minimum benefit for one-person households." +us,scenario_013,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"It found income above Arizona's expanded categorical eligibility limit by using the old 185% FPL threshold. From March 2026 the limit is 200% FPL ($2,608.33/month), which the household's $2,539.33 gross income meets. The asset test is then waived and the $24 minimum applies for March–December." +us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"It applied the ordinary income and asset limits. It missed that Arizona's 200% FPL BBCE from March 2026 deems both met for $2,539.33 gross monthly income, which yields the $24 one-person minimum for ten months." +us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit. It never considered Arizona's 200% FPL BBCE (effective March 2026), which treats the net income test as met and gives the household the $24 minimum allotment for March–December." +us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"It applied the ordinary net income limit and the elderly/disabled resource limit. It missed Arizona's 200% FPL BBCE from March 2026, which deems both met for this household and yields the $24 one-person minimum for ten months." +us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"It correctly found that 30% of net income exceeds the $298 maximum, but it stopped there. It missed that Arizona's 200% FPL BBCE makes the household categorically eligible from March 2026, so the $24 one-person minimum allotment applies for ten months." +us,scenario_013,snap,grok-build-0.1,llm_error,asset_resource,False,"It disqualified the household on the $4,250 elderly/disabled resource limit. Arizona's BBCE at 200% FPL, effective March 2026 and met at $2,539.33/month, eliminates the asset test, so the household gets the $24 minimum for March–December." +us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit with no shelter costs and stopped. It never applied Arizona's 200% FPL BBCE from March 2026, which makes the household categorically eligible and entitles it to the $24 one-person minimum for ten months." +us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the federal elderly/disabled resource limit and the 100% FPL net income test. It missed that Arizona's 200% FPL BBCE (from March 2026) waives both for $2,539.33 gross monthly income, which yields $24 a month for March–December." +us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"It ruled the household out on the net income limit. It never applied Arizona's 200% FPL BBCE (effective March 2026), under which the household is categorically eligible and receives the $24 one-person minimum allotment for ten months." +us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"It applied the ordinary income and asset thresholds. It missed that Arizona's 200% FPL BBCE from March 2026 deems both tests met for this household, which produces the $24 minimum allotment for March–December." +us,scenario_013,snap,muse-wonder,llm_error,categorical_eligibility,False,"It applied the ordinary net income limit and treated the $58,700 in bank assets as disqualifying. Arizona's 200% FPL BBCE, effective March 2026, waives both tests for $2,539.33 gross income and yields the $24 one-person minimum for ten months." +us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit to an elderly/disabled household and stopped. It missed Arizona's 200% FPL BBCE from March 2026, which makes the household categorically eligible for the $24 one-person minimum allotment for March–December." +us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It applied a 130% FPL gross income test, which elderly/disabled households are exempt from. It also missed Arizona's 200% FPL BBCE from March 2026, which makes this $2,539.33/month household categorically eligible for the $24 minimum for ten months." +us,scenario_013,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It invented a rule that SNAP excludes members aged 60 or older. In fact, elderly households are eligible under special rules, and under Arizona's 200% FPL BBCE from March 2026 this household is categorically eligible for the $24 one-person minimum for ten months." us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied Arizona’s 2.5% rate to $88 of dividends and capital gains without subtracting the Arizona standard deduction, which reduces taxable income to zero. It also violated its own computation by submitting $744.10 after its explanation calculated $2.20." -us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Arizona offers no refundable individual credits. Arizona's refundable Credit for Increased Excise Taxes pays $25 to a single filer with AGI of $12,500 or less, and this filer's AGI is $6,736." -us,scenario_013,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the refundable Credit for Increased Excise Taxes. With AGI of $6,736, under the $12,500 single limit, the filer gets $25 for themselves." -us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model checked only the Family Income Tax Credit and the Property Tax Credit, which require dependents or rent/property tax. It missed the Credit for Increased Excise Taxes, which needs only AGI of $12,500 or less for a single filer and gives $25 here." -us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model named the increased excise tax credit but treated it as needing rent or property-tax facts. The credit's only income test is AGI of $12,500 or less for a single filer. This filer's AGI is $6,736, so the $25 credit applies." -us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model assumed the increased excise tax credit pays nothing to a single filer with no dependents and no tax liability. The credit is refundable and gives $25 for the taxpayer alone when AGI is $12,500 or less. This filer's AGI is $6,736." -us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model found the correct credit but counted the age-65 and disability exemptions as extra $25 units, giving $75. The credit is $25 per family member, meaning the taxpayer, spouse, and dependents, so a single filer gets $25. The model also used a $25,000 single limit; the actual limit is $12,500." -us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said Arizona's credits are nonrefundable or need dependents or property tax. It missed the refundable Credit for Increased Excise Taxes, which gives $25 to a single filer with AGI of $12,500 or less; this filer's AGI is $6,736." -us,scenario_013,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model checked only the property tax credit and the family credit. It missed the refundable Credit for Increased Excise Taxes, which needs no rent, property tax, or dependents and gives $25 at AGI of $6,736." -us,scenario_013,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model assumed that having no tax liability and no dependents rules out refundable credits. The Credit for Increased Excise Taxes is refundable and gives $25 to a single filer with AGI of $12,500 or less, regardless of liability." -us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated dependents as required for any Arizona refundable credit. The Credit for Increased Excise Taxes gives $25 for the taxpayer alone when AGI is $12,500 or less, and this filer's AGI is $6,736." -us,scenario_013,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model ruled out credits because there were no dependents, donations, or property tax/rent facts. It missed the refundable Credit for Increased Excise Taxes, which depends only on AGI of $12,500 or less for a single filer and gives $25." -us,scenario_013,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model treated the credit as requiring earned income, like an EITC. Arizona's Credit for Increased Excise Taxes has no earnings requirement and gives $25 to a single filer with AGI of $6,736." -us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the Credit for Increased Excise Taxes. A single filer with AGI of $6,736, under the $12,500 limit, receives $25." -us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said there are no state refundable credits and missed Arizona's refundable Credit for Increased Excise Taxes. That credit gives $25 to this single filer with AGI of $6,736." -us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model said the household qualifies for no refundable credit, missing Arizona's Credit for Increased Excise Taxes. That credit gives $25 per taxpayer when single-filer AGI is $12,500 or less, and AGI here is $6,736." -us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no Arizona refundable credit and skipped the Credit for Increased Excise Taxes. That credit gives $25 to a single filer with AGI of $6,736." -us,scenario_013,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said Arizona has no major refundable credit this person qualifies for. It missed the Credit for Increased Excise Taxes, which gives $25 to a single filer with AGI of $12,500 or less." -us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeds the credit limits. Federal AGI is $6,736 because none of the Social Security is taxable, which is well under the $12,500 single limit for the Credit for Increased Excise Taxes, so the $25 credit applies." -us,scenario_013,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the Credit for Increased Excise Taxes. That credit gives $25 to a single filer with AGI of $6,736, under the $12,500 limit." -us,scenario_013,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said no refundable credits apply and missed Arizona's Credit for Increased Excise Taxes. That credit gives $25 to this single, low-AGI filer." -us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said nothing in the facts triggers a refundable credit, missing Arizona's Credit for Increased Excise Taxes. The facts given, a single filer with AGI of $6,736, are enough to qualify for $25." -us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model looked for dependents or earned income. Arizona's Credit for Increased Excise Taxes needs neither and gives $25 to a single filer with AGI of $12,500 or less; AGI here is $6,736." -us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model found no supported Arizona refundable credit and missed the Credit for Increased Excise Taxes. With $6,736 AGI, a single filer receives $25." -us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model required property tax, rent, or dependents. Arizona's refundable Credit for Increased Excise Taxes depends only on income (AGI of $12,500 or less for a single filer) and gives $25 here." -us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model concluded a one-person elderly household gets no Arizona refundable credit. The Credit for Increased Excise Taxes gives $25 to a single taxpayer with AGI of $6,736." -us,scenario_013,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model said no refundable Arizona credit is supported, missing the Credit for Increased Excise Taxes. AGI of $6,736 is under the $12,500 single limit, so the credit is $25." -us,scenario_013,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model required earnings, dependents, or expenses. Arizona's Credit for Increased Excise Taxes needs none of these and gives $25 to a single filer with AGI of $12,500 or less." -us,scenario_013,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model looked at the Working Poor Tax Credit, which is a nonrefundable donation credit, and missed the refundable Credit for Increased Excise Taxes. That credit gives $25 to this single filer with AGI of $6,736." -us,scenario_013,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model checked only the property tax credit and missed the Credit for Increased Excise Taxes. That credit needs no property tax or other expenses and gives $25 at AGI of $6,736." -us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said household composition and income rule out refundable credits. A single filer with AGI of $6,736 meets the $12,500 limit for Arizona's Credit for Increased Excise Taxes, which pays $25." -us,scenario_013,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model treated zero taxable income and no dependents as disqualifying. The Credit for Increased Excise Taxes is refundable and gives $25 for the taxpayer alone when AGI is $12,500 or less." -us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model looked only at EITC-style and property tax credits. Arizona's Credit for Increased Excise Taxes has no earned-income or expense requirement and gives $25 at AGI of $6,736." -us,scenario_013,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model tested $30,544 of gross household income, including non-taxable Social Security, against a $25,000 single limit. The credit tests federal AGI, which is $6,736 because no Social Security is taxable here, against a $12,500 single limit, so the $25 credit applies." -us,scenario_013,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model treated dependents as required for Arizona refundable credits. The Credit for Increased Excise Taxes gives $25 for the taxpayer when single-filer AGI is $12,500 or less." -us,scenario_013,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model applied the Property Tax Credit's income limit and ignored the Credit for Increased Excise Taxes, a separate credit. That credit tests AGI ($6,736 here) against $12,500 and gives $25." -us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model required qualifying children or earned income. Arizona's Credit for Increased Excise Taxes needs neither and gives $25 to a single filer with AGI of $6,736." -us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model assumed a refundable credit needs tax liability to offset. Arizona's Credit for Increased Excise Taxes is refundable and gives $25 to a single filer with AGI of $12,500 or less, even with zero liability." +us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Arizona offers no refundable individual income tax credits. It missed the refundable Credit for Increased Excise Taxes, which pays $25 per tax-unit member to single filers with Arizona AGI at or below $12,500. This filer's Arizona AGI of about $6,736 qualifies for $25." +us,scenario_013,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and never considered the refundable Credit for Increased Excise Taxes. With Social Security excluded, Arizona AGI is about $6,736, under the $12,500 single limit, so the one-person unit gets $25." +us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model checked only the Family Income Tax Credit and the Property Tax Credit, which need dependents or rent/property tax. It skipped the refundable Credit for Increased Excise Taxes, which needs none of those: $25 per person for single filers with Arizona AGI at or below $12,500. Here Arizona AGI is about $6,736, so the credit is $25." +us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model named the increased excise tax credit and found that Arizona taxable income was very low, but still said its conditions were not met. The only test is Arizona AGI at or below $12,500 for a single filer, with no rent, property tax, or dependent requirement. Arizona AGI of about $6,736 passes, so the filer gets $25." +us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model said the increased excise tax credit pays nothing to a single filer with no dependents and no tax liability. The credit is refundable and pays $25 for the taxpayer alone whenever Arizona AGI is at or below $12,500. Liability and dependents do not matter, so this filer gets $25." +us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model found the right credit and correctly put Arizona AGI well under the limit. It then multiplied $25 by three exemptions: personal, age 65+, and disability. The Credit for Increased Excise Taxes counts only the taxpayer, spouse, and dependents, not age or disability exemptions, so a single filer gets $25, not $75. The model also used a $25,000 single-filer limit; the actual limit is $12,500." +us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model looked only at the family income tax credit and the property tax credit and concluded nothing applied. It missed the refundable Credit for Increased Excise Taxes, which pays $25 to a single filer with Arizona AGI at or below $12,500. At about $6,736, this filer gets $25." +us,scenario_013,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model ruled out only the property tax credit (no rent or property tax) and the family credit (no dependents). It skipped the refundable Credit for Increased Excise Taxes, which needs neither and pays $25 to a single filer with Arizona AGI of about $6,736." +us,scenario_013,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model cited zero state tax liability and no dependents as reasons for $0. The refundable Credit for Increased Excise Taxes depends on neither: it pays $25 for the taxpayer alone when Arizona AGI is at or below $12,500, as this filer's $6,736 is." +us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated having no dependents as disqualifying. The Credit for Increased Excise Taxes pays $25 per tax-unit member, including a single taxpayer with no dependents, when Arizona AGI is at or below $12,500. Here that gives $25." +us,scenario_013,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model required dependents, donations, or property tax/rent. The refundable Credit for Increased Excise Taxes requires none of these and is based only on Arizona AGI at or below $12,500 for single filers. This filer's AGI of about $6,736 gives $25." +us,scenario_013,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model treated Arizona's refundable credits as depending on earned income, like an EITC. The Credit for Increased Excise Taxes has no earned-income requirement: unearned pension and investment income count toward Arizona AGI, and at about $6,736 that AGI qualifies for $25." +us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and missed the Credit for Increased Excise Taxes. Arizona AGI of about $6,736 is under the $12,500 single-filer limit, so the one-person unit gets $25." +us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no state refundable credits apply and missed Arizona's refundable Credit for Increased Excise Taxes. That credit pays $25 to a single filer whose Arizona AGI, about $6,736 here, is at or below $12,500." +us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model said the household qualifies for no state refundable credits. It missed the Credit for Increased Excise Taxes, which pays $25 because this single filer's Arizona AGI of about $6,736 is under the $12,500 limit." +us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no Arizona refundable credit and overlooked the Credit for Increased Excise Taxes. The credit pays $25 per tax-unit member to single filers with Arizona AGI at or below $12,500; this filer is at about $6,736." +us,scenario_013,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said Arizona has no major refundable credit this filer qualifies for. The refundable Credit for Increased Excise Taxes applies to a single filer with Arizona AGI of about $6,736 and pays $25." +us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model said household income exceeds the limits for Arizona refundable credits. The limit applies to Arizona AGI, not gross income. Social Security is excluded (none is federally taxable at provisional income of about $18,676, and Arizona exempts it anyway), so Arizona AGI is about $6,736, under the $12,500 excise-credit limit, which gives $25." +us,scenario_013,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model said no Arizona refundable credit is available and missed the Credit for Increased Excise Taxes. That credit pays $25 to this single filer because Arizona AGI of about $6,736 is at or below $12,500." +us,scenario_013,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and overlooked the Credit for Increased Excise Taxes. The credit gives $25 to a one-person unit with Arizona AGI at or below $12,500; here Arizona AGI is about $6,736." +us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said no Arizona refundable credit is triggered. The Credit for Increased Excise Taxes is triggered by low Arizona AGI alone, and this single filer's AGI of about $6,736 is under $12,500, so the credit is $25." +us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model looked for dependents or earned income as triggers. The Credit for Increased Excise Taxes requires neither: it pays $25 for the taxpayer when Arizona AGI is at or below $12,500, as this filer's $6,736 is." +us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model found no supported Arizona refundable credit. The facts given (single filer, Social Security excluded, pension and investment income of about $6,736) are enough to qualify for the $25 Credit for Increased Excise Taxes under the $12,500 Arizona AGI limit." +us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model required property tax, rent, dependents, or other credit-generating facts. The Credit for Increased Excise Taxes needs only Arizona AGI at or below $12,500 for a single filer, and this filer's $6,736 gives $25." +us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model found no refundable credit eligibility for a one-person elderly household. A one-person unit with Arizona AGI of about $6,736 qualifies for the Credit for Increased Excise Taxes at $25 per person, so the credit is $25." +us,scenario_013,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model found no supported Arizona refundable credit and missed the Credit for Increased Excise Taxes. With Social Security excluded, this single filer's Arizona AGI is about $6,736, under the $12,500 limit, so the credit is $25." +us,scenario_013,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"The model treated earnings, dependents, or expenses as required. The Credit for Increased Excise Taxes is based only on Arizona AGI at or below $12,500 for a single filer and pays $25 per tax-unit member, so this filer gets $25." +us,scenario_013,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model checked only the Working Poor Tax Credit, which is a nonrefundable donation credit. It missed the refundable Credit for Increased Excise Taxes, which pays $25 to a single filer with Arizona AGI of about $6,736, under the $12,500 limit." +us,scenario_013,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model looked only at the property tax credit, which needs property tax or rent. It ignored the refundable Credit for Increased Excise Taxes, which needs no expenses and pays $25 to a single filer with Arizona AGI of about $6,736." +us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said household composition and income level rule out state refundable credits. A one-person unit with Arizona AGI of about $6,736 is exactly who the $25 Credit for Increased Excise Taxes covers, since the single-filer limit is $12,500." +us,scenario_013,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model cited zero taxable income and no qualifying dependents as reasons for $0. The Credit for Increased Excise Taxes is refundable, needs no tax liability or dependents, and pays $25 to a single filer with Arizona AGI at or below $12,500, as here." +us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model looked for a refundable EITC or property tax credit and stopped there. Arizona's refundable Credit for Increased Excise Taxes has no earned-income or expense requirement and pays $25 to this single filer, whose Arizona AGI is about $6,736." +us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model tested the excise credit against total income of $30,544, which includes $23,736 of Social Security. The test uses Arizona AGI. Social Security is not federally taxable at provisional income of about $18,676 and is exempt in Arizona anyway, so Arizona AGI is about $6,736. That is under the single-filer limit (actually $12,500, not the $25,000 the model cited) and gives $25." +us,scenario_013,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated having no dependents as disqualifying. The Credit for Increased Excise Taxes pays $25 for the taxpayer alone when Arizona AGI is at or below $12,500, so this single filer with about $6,736 of Arizona AGI gets $25." +us,scenario_013,state_refundable_credits,muse-wonder,llm_error,categorical_eligibility,False,"The model required qualifying children or earned income. The Credit for Increased Excise Taxes requires neither and pays $25 to a single filer whose Arizona AGI, here about $6,736 of pension and investment income, is at or below $12,500." +us,scenario_013,state_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model tested only the property tax refund credit, using gross income of $30,544. It missed the separate refundable Credit for Increased Excise Taxes, which uses Arizona AGI. Social Security is excluded, leaving about $6,736, under the $12,500 single limit, which gives $25." +us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model required qualifying children or earned income and checked only an EITC and family credit. The Credit for Increased Excise Taxes pays $25 per tax-unit member, including a lone taxpayer with only unearned income, when Arizona AGI is at or below $12,500, as this filer's $6,736 is." +us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model reasoned that with no Arizona tax liability there was nothing to offset. The Credit for Increased Excise Taxes is refundable and paid regardless of liability: a single filer with Arizona AGI of about $6,736, under the $12,500 limit, gets $25." us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premiums from the gross wage input, which cut AGI to $74,211 instead of $88,927.65. It then submitted $5,647, which matches neither its own $4,545 calculation nor the correct $56,727.65 × 2026 bracket result of $6,311.32." us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $28,000 standard deduction instead of the 2026 MFJ amount of $32,200. It also taxed all of its taxable income at 12% and skipped the 10% bracket on the first $24,800. Finally, it replaced its own $7,311 result with an unexplained $6,819." us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model gave a rough '12% bracket' estimate of $6,900 and never actually ran the brackets. The correct calculation subtracts the $32,200 standard deduction, takes 10% of the first $24,800, and takes 12% of the remaining $31,927.65, for $6,311.32." @@ -1143,53 +1160,54 @@ us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there is no answer to grade. The correct derivation is 7.65% of the spouse's $88,928 wages, about $6,803." us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model's explanation computes $5,513.54 Social Security plus $1,289.46 Medicare, about $6,803, on the spouse's $88,928 wages. It then submitted 'value = 6581', a number that contradicts its own sum." us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct components, $5,513.54 Social Security and $1,289.46 Medicare on $88,928 with no Additional Medicare Tax. It then submitted $8,075.02 instead of their $6,802.99 sum, adding about $1,272 that its own explanation never supports." -us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the $84,928 base right and computed about $3,256 using the 2025 WV rates. It then raised the figure for no stated reason to $3,374. The 2026 schedule is about 5% below the 2025 rates, so the tax is $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The taxable income of $84,928 is correct. The model's 2026 rates (2.13/2.84/3.20/4.26/4.63%) are only about 4% below the 2025 schedule. The actual 2026 cut is 5% (about 2.11/2.81/3.16/4.22/4.58%), so it overstated the tax by about $33." -us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, which does not reduce AGI here. It also skipped the two $2,000 WV personal exemptions and applied a made-up ~4.5% average rate instead of the graduated 2026 MFJ brackets on $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model gave no derivation and a rough $2,700. The 2026 graduated MFJ brackets (about 2.11% to 4.58%) applied to $88,928 AGI minus $4,000 in exemptions produce $3,092. The model understated that by about $390." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model computed $3,255 on the correct $84,928 base using the 2025 rates. It then cut that to $2,877 for unnamed nonrefundable credits, even though none apply. It never applied the 2026 rates, which are another 5% lower and give $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The base is correct, but the model mixed 2025-era rates with invented ones (4.07%, 4.625%) to reach about $3,130. It then applied an arbitrary 'further cut' down to $2,849 instead of using the 2026 schedule, which yields $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income, but $1,655 is only about half of the tax the 2026 graduated brackets produce on that base ($3,092). It also cited nonrefundable credits, but none apply." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model applied the 2025 WV rates (2.22% to 4.82%) to the correct $84,928 base. It missed the further 5% cut for 2026, which lowers the brackets to about 2.11% to 4.58% and the tax to $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an outdated four-bracket schedule (2.36/3.15/3.54/4.72%) to get about $3,360. It then dropped to $1,919 by assuming credits that it had itself just said do not apply. The 2026 five-bracket schedule on $84,928 gives $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but used a 2023–2025 rate range topping out at 4.82%, then rounded to $3,200. It missed the 2026 schedule, which tops out at about 4.58% and yields $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model assumed the 2024 brackets (2.36% to 5.12%) still apply in 2026. Two rounds of rate cuts have lowered them to about 2.11% to 4.58%, so the tax on $84,928 is $3,092, not $3,460." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model invented a $4,000 WV standard deduction, which does not exist, and $6,000 personal exemptions per person, when the exemption is $2,000 each. That cut taxable income to $72,928 instead of $84,927.65. It also used made-up brackets rather than the 2026 schedule." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, giving an AGI of $74,211 instead of $88,928. It then took a $15,400 standard deduction, which WV does not allow, skipped the $2,000 personal exemptions, and applied invented rates up to 5.12%." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The rates it used are close to the 2026 schedule. But the model cut the base by subtracting the $14,717 ESI premium (AGI $74,211) and adding a nonexistent $4,000 WV standard deduction, which left $66,211 instead of $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model subtracted $8,000 instead of the correct $4,000 in personal exemptions (WV has no standard deduction). It also applied made-up rates from 1.69% to 4.29%, well below the 2026 schedule of about 2.11% to 4.58%." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium, giving an AGI of $74,211 instead of $88,928, so taxable income was $70,211 instead of $84,927.65. It also applied the 2024 brackets (2.36% to 5.12%) rather than the lower 2026 schedule." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no derivation. $4,125 is about 4.86% of the full $84,928, which is a flat top-bracket-style rate. WV taxes this income with graduated 2026 brackets (about 2.11% to 4.58%), producing $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the $14,717 ESI premium as a reduction to wages, giving $74,211 of AGI and $70,211 of taxable income instead of $88,928 and $84,927.65. It then applied the 2024 rates instead of the 2026 schedule." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, giving an AGI of $74,211 and taxable income of $70,211 instead of $84,927.65. Its result also matches the 2024 brackets rather than the 2026 rates, which are about 5% below the 2025 schedule." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model claimed deductions and exemptions wipe out the liability. WV has no standard deduction, and the only subtraction is $4,000 of personal exemptions, which leaves $84,927.65 taxable and $3,092 of tax." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI by the $14,717 ESI premium, giving $74,211 instead of $88,928, and applied the 2024 brackets to $70,211. The correct computation applies the 2026 schedule to $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $74,211 by subtracting the $14,717 ESI premium, which understated taxable income by $14,717. It then applied an unspecified rate schedule that is higher than the 2026 brackets." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but applied the 2024 brackets (2.36/3.15/3.54/4.72/5.12%). It missed the 2025 cut and the further 5% 2026 cut that bring the tax to $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an $8,000 WV standard deduction, which does not exist. It then applied the 4.82% top rate flat to $76,928 instead of running $84,927.65 through the graduated 2026 brackets." -us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model computed Kentucky income tax and a Kentucky family size credit for a West Virginia household. The WV computation ($88,928 AGI minus $4,000 in exemptions, run through the 2026 MFJ brackets) yields $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no derivation. $2,331 is about 75% of the correct liability, which is consistent with a reduced base or rates far below the 2026 WV MFJ schedule applied to $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and credits eliminate the WV tax. WV allows only $4,000 of personal exemptions for this couple, and no nonrefundable credit applies at $88,928 of AGI, so the tax is $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but applied the 2025 brackets (2.22% to 4.82%) as if they were the 2026 schedule. The 2026 rates are another 5% lower, which reduces the tax from $3,255 to $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The base is correct ($84,928). The model's rate schedule produces $3,217.53, only about 1% below the 2025 schedule. The full 5% 2026 cut gives $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model applied the 2025 WV rates to the correct $84,928, getting $3,255.03. It missed the 2026 cut of about 5%, which lowers every bracket and brings the tax to $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's $3,255 is the 2025 rate schedule applied to the correct $84,928 base. It did not apply the further 5% rate reduction for 2026, which yields $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model used the 2025 brackets (2.22/2.96/3.33/4.44/4.82%) for tax year 2026. The 2026 brackets are 5% lower (about 2.11/2.81/3.16/4.22/4.58%), so the tax on $84,928 is $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The base is correct ($84,928), but the model's schedule is only about 2% below the 2025 rates. The 2026 brackets are about 5% below them (roughly 2.11% to 4.58%), which yields $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The taxable income is correct, but the model applied the 2025 joint rates (2.22% to 4.82%). It missed the 2026 reduction to about 2.11% to 4.58%, which yields $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The base is correct ($84,928), but the model's 2026 rates are about 4.5% below the real 2026 brackets. That understates the tax by about $140 compared with the schedule that is 5% below the 2025 rates, which gives $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied a standard deduction that WV does not have and concluded the tax is zero. WV subtracts only $2,000 per exemption, which leaves $84,927.65 taxable and $3,092 of tax." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed the 2024 brackets (2.36% to 5.12%) are still the 2026 baseline. The 2025 and 2026 cuts lower them to about 2.11% to 4.58%, so the tax on $84,928 is $3,092, not $3,460." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted about $20,600 of made-up deductions and exemptions, when WV allows only $4,000 of personal exemptions and no standard deduction. It also applied the 2024 rates (2.36% to 5.12%) instead of the 2026 schedule." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model assumed its estimated 2025 rates (2.27% to 4.92%) carry into 2026. Those are already above the actual 2025 rates, and the 2026 brackets are another 5% lower (about 2.11% to 4.58%), giving $3,092 on $84,928." -us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from AGI and then a $30,840 federal standard deduction, which WV does not allow. WV starts from federal AGI and subtracts only $2,000 per exemption. The model also used outdated 2%–5% rates." -us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The base is correct (about $84,900), but $3,422 matches the higher pre-cut 2024-era rates. The 2026 brackets (about 2.11% to 4.58%) yield $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium (AGI $74,211) and an $8,000 WV standard deduction that does not exist. It then applied a nonexistent flat 4.5% rate instead of the graduated 2026 MFJ brackets on $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used the 2024 schedule ($2,183.50 plus 5.12% above $60,000) on the correct $84,928. It missed the 2025 and 2026 rate reductions that bring the top rate to about 4.58% and the tax to $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed the personal exemption reduces taxable wages to zero. The WV exemption is only $2,000 per person ($4,000 total), which leaves $84,927.65 taxable and $3,092 of tax." -us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly arrived at $84,928 of taxable income but mixed in 2025 rates (including the 4.82% top rate) and inconsistent bracket amounts. The 2026 brackets, 5% below 2025, yield $3,092." -us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium to get $74,211 of AGI and built an inverted bracket schedule with rates falling as income rises. It also applied a 4% 'family tax credit' that does not apply at this income. The correct computation runs $84,927.65 through the ascending 2026 MFJ brackets." -us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model applied the federal standard deduction and assumed nonrefundable credits eliminate the tax. WV allows no standard deduction, only $4,000 of exemptions, and no credit applies here, so $84,927.65 is taxed for $3,092." +us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly found $84,928 of taxable income but applied the 2025 WV rates (2.22–4.82%), getting about $3,256. It then raised that to $3,374 for no stated reason. The 2026 schedule (2.11/2.81/3.16/4.22/4.58%) gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The taxable income of $84,928 was right, but the rates it assumed (2.13/2.84/3.20/4.26/4.63%) reflect only about a 4% cut from 2025. The 2026 law cut a full 5%, to 2.11/2.81/3.16/4.22/4.58%, which gives $3,092.19, not $3,125.17." +us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $14,717 ESI premium from wages, and its arithmetic was also wrong ($74,411 instead of $74,211). It never took the two $2,000 WV personal exemptions and applied a made-up ~4.5% average rate instead of the graduated schedule. The correct base is $84,927.65, taxed at 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave a rounded $2,700 guess with no calculation. The correct path, $88,927.65 AGI minus $4,000 in exemptions = $84,927.65 taxed under the 2026 WV MFJ brackets (2.11–4.58%), gives $3,092.19. So $2,700 understates the tax at the right rates." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It computed $3,255 using the 2025 rates (2.22–4.82%) instead of the 2026 rates (2.11–4.58%). It then subtracted about $378 for nonrefundable credits that it had already said do not apply, ending at $2,877." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied a mix of rates (2.2%, 2.96%, 3.33%, 4.07%, 4.625%) that matches no enacted WV schedule, totaling $3,130. It then applied an arbitrary further cut to reach $2,849. The 2026 schedule of 2.11/2.81/3.16/4.22/4.58% on $84,927.65 gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The $84,928 taxable income was right, but $1,655 is roughly half the tax that the 2026 brackets produce on that amount. The model misapplied the rate schedule; 2.11/2.81/3.16/4.22/4.58% across the $10k/$25k/$40k/$60k breakpoints gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It applied the 2025 WV rates (2.22–4.82%) to the correct $84,928 base, producing $3,255.03. It missed the further 5% rate cut for 2026, to 2.11–4.58%, which gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a four-bracket schedule (2.36–4.72%) with no $60k breakpoint and got $3,360. It then reported $1,919 by assuming unexplained credits and further cuts, even though it had said no credits apply. The 2026 schedule (2.11/2.81/3.16/4.22/4.58%) on $84,927.65 gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The $84,928 base was right, but it estimated the tax with a vague blend of rates (2.36% to a 4.82–5.12% top rate) rather than the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%. That overstated the tax at about $3,200 instead of $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It explicitly assumed the 2024 rates (2.36/3.15/3.54/4.72/5.12%) still apply in 2026. It missed both the 2025 cut and the further 2026 cut to 2.11/2.81/3.16/4.22/4.58%, which gives $3,092.19 instead of $3,459.80." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a $4,000 WV standard deduction and $6,000 exemptions per person. WV has no standard deduction and allows $2,000 per exemption, so taxable income is $84,927.65, not $72,928. It also used outdated, distorted rates (2.36–5.16%) instead of the 2026 schedule of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It started from $74,211 by subtracting the ESI premium, which is not deducted again, so AGI is $88,927.65. It then subtracted a nonexistent $15,400 WV standard deduction instead of two $2,000 exemptions, and used a made-up three-rate schedule topping out at 5.12%." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It cut AGI to $74,211 by subtracting the $14,717 ESI premium and added a nonexistent $4,000 WV standard deduction, giving $66,211 instead of $84,927.65. Its rates (2.18–4.72%) were also slightly above the 2026 schedule of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It subtracted $8,000 of deductions and exemptions, but WV allows only $4,000 in exemptions, so taxable income is $84,927.65. It also applied rates (1.69–4.29%) well below the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium to get $74,211 AGI, but AGI is the full $88,927.65, which makes taxable income $84,927.65, not $70,211. It also used the 2024 rates topping out at 5.12% instead of the 2026 rates of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave $4,125 with no calculation, which implies an effective rate of about 4.9% on $84,928, consistent with a flat top-bracket rate. WV taxes in graduated brackets of 2.11/2.81/3.16/4.22/4.58% in 2026, which gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium from wages to get $74,211 and taxed $70,211. AGI is $88,927.65, so taxable income is $84,927.65. Its $2,706 also reflects the 2024 rates (2.36–5.12%), not the 2026 rates of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $74,211 by subtracting the $14,717 ESI premium, giving taxable income of $70,211 instead of $84,927.65. It also applied the 2024 rate schedule (top rate 5.12%) rather than the 2026 rates of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed standard deductions and exemptions wipe out the tax. WV has no standard deduction and allows only $4,000 in exemptions here, leaving $84,927.65 taxable, which produces $3,092.19 at the 2026 rates." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It started from $74,211 by subtracting the ESI premium, so it taxed $70,211 instead of $84,927.65. Its $2,706.30 matches the 2024 schedule (2.36–5.12%), not the 2026 schedule of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium to get $74,211 AGI and $70,211 taxable income. The correct base is $88,927.65 − $4,000 = $84,927.65, taxed at 2.11/2.81/3.16/4.22/4.58% for $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The $84,928 base was right, but it applied the 2024 rates (2.36/3.15/3.54/4.72/5.12%). It missed the 2025 and 2026 cuts down to 2.11/2.81/3.16/4.22/4.58%, which give $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It invented an $8,000 WV standard deduction, which does not exist, and applied the 4.82% top rate flat to all of taxable income instead of using graduated brackets. The correct calculation taxes $84,927.65 through the 2026 brackets of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It computed Kentucky income tax with a Kentucky family size credit, using an unrelated $25,050 taxable income, for a West Virginia household. WV taxes $84,927.65 at the 2026 rates of 2.11–4.58%, which gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave $2,331 with no calculation, which understates the tax on $84,927.65 under the 2026 WV MFJ brackets (2.11/2.81/3.16/4.22/4.58%). Those brackets give $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and nonrefundable items eliminate the tax. WV allows only two $2,000 exemptions and no standard deduction, and no nonrefundable credit applies at this income, so $84,927.65 is taxable and produces $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the 2025 WV rates (2.22/2.96/3.33/4.44/4.82%) to the correct $84,928 base, giving $3,255.03. It missed the further 5% cut for 2026, to 2.11/2.81/3.16/4.22/4.58%, which gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The $84,928 taxable income was right, but its $3,217.53 comes from a rate schedule above the 2026 WV rates. The 2.11/2.81/3.16/4.22/4.58% schedule gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $3,255.03 is the 2025 schedule (2.22–4.82%) applied to $84,928. It did not apply the 2026 5% rate cut to 2.11–4.58%, which gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $3,255 equals the 2025 WV rates (2.22–4.82%) on $84,928 of taxable income. The 2026 rates of 2.11/2.81/3.16/4.22/4.58% give $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It explicitly used the 2025 rates of 2.22/2.96/3.33/4.44/4.82% on $84,928. It missed the 2026 cut to 2.11/2.81/3.16/4.22/4.58%, which gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The $84,928 base was right, but its $3,192.03 reflects rates partway between the 2025 and 2026 schedules. The full 2026 schedule of 2.11/2.81/3.16/4.22/4.58% gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied the 2025 joint rates of 2.22/2.96/3.33/4.44/4.82% to $84,928, giving $3,255.03. The 2026 rates, cut another 5% to 2.11/2.81/3.16/4.22/4.58%, give $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The $84,928 base was right, but its $2,952.20 implies rates below the 2026 WV schedule. The 2.11/2.81/3.16/4.22/4.58% brackets on $84,927.65 give $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied a standard deduction that WV does not have and concluded the tax is zero. Only $4,000 in exemptions comes off $88,927.65 AGI, leaving $84,927.65 taxable and $3,092.19 of tax." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the 2024 brackets (2.36/3.15/3.54/4.72/5.12%) are still the 2026 baseline. It missed the 2025 and 2026 cuts to 2.11/2.81/3.16/4.22/4.58%, which give $3,092.19 instead of $3,460." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted about $20,600 of made-up deductions and exemptions. WV allows only two $2,000 exemptions, for $84,927.65 of taxable income. It also used the outdated 2.36–5.12% rates instead of the 2026 rates of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It carried rates of 2.27/3.02/3.40/4.53/4.92% into 2026. Those are the 2024 rates cut 4%, not the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%, which gives $3,092.19 rather than $3,322.46." +us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium to get $74,211 AGI and then took the federal standard deduction of $30,840, which WV does not allow. It also used made-up 2/3/4/5% rates. WV taxable income is $84,927.65 after only $4,000 in exemptions, taxed at 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It had about $84,900 of taxable income, correctly, but its $3,422 implies the pre-2025 rate schedule (top rate near 5.12%). The 2026 rates of 2.11/2.81/3.16/4.22/4.58% give $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium from wages and invented an $8,000 WV standard deduction, ending at $62,211 instead of $84,927.65. It then applied a flat 4.5% rate, but WV still uses graduated brackets (2.11–4.58% in 2026)." +us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the 2024 brackets, $2,183.50 plus 5.12% above $60k, to $84,928. The 2026 brackets of 2.11/2.81/3.16/4.22/4.58% produce $1,950.50 plus 4.58% of the excess over $60k, for $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed the personal exemption reduces taxable wages to zero. The WV exemption is only $2,000 per person ($4,000 total), leaving $84,927.65 taxable and $3,092.19 of tax." +us,scenario_014,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used the 2025 schedule with a 4.82% top rate on $84,928, giving $3,255.03. The 2026 schedule caps at 4.58% (2.11/2.81/3.16/4.22/4.58%) and gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used inconsistent bracket amounts ($222, $438, $552, $854) with a 4.82% top rate. Neither matches the 2026 schedule, which gives $211 + $421.50 + $474 + $844 + 4.58% × $24,927.65 = $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It subtracted the ESI premium to get $74,211 AGI and used a made-up rate schedule that runs backwards, from 4.80% down to 2.88%. It then applied a 4% family tax credit that does not apply at this income. The correct calculation taxes $84,927.65 at 2.11% rising to 4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the federal standard deduction, which WV does not allow, and assumed nonrefundable credits wipe out the tax. WV taxable income is $84,927.65 after $4,000 in exemptions, and no nonrefundable credit applies, so the tax is $3,092.19." us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning matches the reference exactly: AGI $42,909, taxable income $26,809, and tax of about $2,969 at 10% to $12,400 and 12% above. It then submitted $3,103 with no derivation. That $134 overstatement is a transcription error, not a different rule." us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It counted the $960 of nontaxable workers' compensation as income and deducted the full $4,510 capital loss instead of the $3,000 cap. It also used the 2024 single standard deduction ($14,600) and 2024 10% bracket ($11,600) instead of the 2026 values ($16,100 and $12,400). It then dropped its own $3,094 result and submitted an unexplained $3,509." us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It applied 'preferential capital gain treatment' even though the net capital result is a loss, capped at $3,000 against ordinary income, so every dollar of the $26,809 taxable income is taxed at ordinary rates. Applying 10%/12% to that amount gives $2,969. The $2,900 answer reflects understated income or misapplied rates." @@ -1235,53 +1253,54 @@ us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,age_disability,F us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model decided that being disabled qualifies an adult under 65 for Medicare. That skips the requirement of 24 months of Social Security Disability Insurance entitlement. The 36-year-old head has no SSDI income and no ESRD or ALS status, so they are not Medicare eligible." us,scenario_015,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model based eligibility only on the head being listed as disabled. Disability-based Medicare requires 24 months of SSDI benefit receipt, and a disability flag alone doesn't meet that. The head is 36 and gets no Social Security disability benefits, so the answer is not eligible (0)." us,scenario_015,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated the head's disabled status as enough for Medicare. Under-65 eligibility requires 24 months of SSDI entitlement, or ESRD or ALS status. The head is 36 and has none of these, so Medicare eligibility is 0." -us,scenario_015,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model treated the four-city list (NYC/Philadelphia/KC/St. Louis) as the full scope. The other reading counts every local income tax PolicyEngine models, including Indiana county tax. Under that reading, the default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55." -us,scenario_015,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana county taxes out of the output because they were not in the four-city list. PolicyEngine's local_income_tax includes Indiana county income tax: Adams County's 1.6% on $41,909.38 of Indiana AGI is $670.55. Excluding it under the enumerated-list reading produced $0." -us,scenario_015,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model wrongly said Indiana has no local income tax. All 92 Indiana counties levy a county income tax on Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38 ($42,909.38 federal AGI minus the $1,000 exemption), the tax is $670.55." -us,scenario_015,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model decided Indiana county income tax falls outside the output because only four cities were named. PolicyEngine counts the county tax in local_income_tax, applying Adams County's 1.6% to Indiana AGI of $41,909.38, which gives $670.55." -us,scenario_015,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model read the four named cities as the only taxes in this output and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax. The default Adams County rate of 1.6% on $41,909.38 of Indiana AGI gives $670.55." -us,scenario_015,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model took the output to cover only NYC, Philadelphia, KC and St. Louis taxes, so it skipped Indiana's county income tax. PolicyEngine includes that tax: 1.6% (Adams County) on Indiana AGI of $41,909.38 is $670.55." -us,scenario_015,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model left Indiana county income tax out because of the four-city list. PolicyEngine's local_income_tax includes it and, with no county given, uses Adams County's 1.6% rate on Indiana AGI of $41,909.38, which gives $670.55." -us,scenario_015,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to the named jurisdictions and found none that applied. It never added Indiana county income tax, which PolicyEngine includes in local_income_tax: Adams County's 1.6% times $41,909.38 of Indiana AGI is $670.55." -us,scenario_015,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and concluded nothing applies to an Indiana resident. It missed Indiana county income tax, which PolicyEngine counts in local_income_tax at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55." -us,scenario_015,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model treated the four named city and earnings taxes as the full scope and ignored Indiana's county income tax. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $41,909.38 ($42,909.38 federal AGI minus the $1,000 exemption), giving $670.55." -us,scenario_015,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model read the four cities as the only jurisdictions covered and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% (Adams County) on $41,909.38 is $670.55." -us,scenario_015,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is not in one of the four named cities. It skipped Indiana county income tax, which PolicyEngine includes at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55." -us,scenario_015,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model applied only the four-city list and left out Indiana's county income tax. PolicyEngine computes that tax at the default Adams County rate of 1.6% on $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model checked only the NYC/Philadelphia/KC/St. Louis taxes and missed Indiana county income tax. PolicyEngine includes that tax in local_income_tax: Adams County's 1.6% on Indiana AGI of $41,909.38 is $670.55." -us,scenario_015,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies in Indiana, but every Indiana county levies an income tax on Indiana AGI. The default Adams County rate of 1.6% on $41,909.38 gives $670.55." -us,scenario_015,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model recognized that Indiana has county income taxes but excluded them because of the four-city list. PolicyEngine counts them in local_income_tax, applying Adams County's 1.6% to Indiana AGI of $41,909.38, which gives $670.55." -us,scenario_015,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies and never considered Indiana's county income tax. PolicyEngine levies it at Adams County's 1.6% on Indiana AGI of $41,909.38 (federal AGI of $42,909.38 minus the $1,000 exemption), giving $670.55." -us,scenario_015,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four named city taxes and excluded Indiana county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It missed Indiana county income tax, which PolicyEngine computes at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55." -us,scenario_015,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model said Indiana has no local income tax, but Indiana county income tax applies statewide to Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38, the tax is $670.55." -us,scenario_015,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model treated the four named cities as the only local taxes and left out Indiana county income tax. PolicyEngine includes it in local_income_tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model assumed PolicyEngine models only NYC, Philadelphia, KC and St. Louis local taxes. PolicyEngine also models Indiana county income tax and applies Adams County's 1.6% to Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model read the four-city list as the full output set and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% (Adams County default) on Indiana AGI of $41,909.38 is $670.55." -us,scenario_015,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because Indiana is not among the four listed jurisdictions. It skipped Indiana county income tax, which PolicyEngine computes at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model applied only the four listed city taxes and left out Indiana's county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model found no listed jurisdiction and answered $0, never considering Indiana county income tax. PolicyEngine counts that tax in local_income_tax: Adams County's 1.6% times $41,909.38 of Indiana AGI is $670.55." -us,scenario_015,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model treated the missing county as meaning no local tax. PolicyEngine instead assigns a default county (Adams) and applies its 1.6% county income tax rate to Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC, Philadelphia, KC and St. Louis taxes and excluded Indiana county income tax. PolicyEngine includes that tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model read the listed jurisdictions as exhaustive and, with no county given, answered $0. PolicyEngine defaults to Adams County and applies its 1.6% county income tax to Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model assumed only the four separately modeled city taxes feed this output. PolicyEngine's local_income_tax also includes Indiana county income tax, computed at Adams County's 1.6% on $41,909.38, which gives $670.55." -us,scenario_015,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model found Indiana absent from the listed local-tax jurisdictions and answered $0. PolicyEngine levies Indiana county income tax at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model limited the output to the named cities and left out Indiana county income tax. PolicyEngine includes that tax in local_income_tax at Adams County's 1.6% of $41,909.38 Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"The model checked only the NYC/Philadelphia/KC/St. Louis taxes and answered $0. PolicyEngine also computes Indiana county income tax: 1.6% (Adams County) on Indiana AGI of $41,909.38 is $670.55." -us,scenario_015,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is outside the four named cities. It missed Indiana county income tax, which PolicyEngine includes at Adams County's 1.6% on $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"The model read the output as covering only the listed cities and left out Indiana's county income tax. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies in Indiana, overlooking the county income tax every Indiana county levies on Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38, the tax is $670.55." -us,scenario_015,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model applied only the four listed city taxes and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, which gives $670.55." -us,scenario_015,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model treated NYC, Philadelphia, KC and St. Louis as the only covered taxes and excluded Indiana county income tax. PolicyEngine computes that tax at the default Adams County rate of 1.6% on $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is not in one of the four named cities. It skipped Indiana county income tax, which PolicyEngine levies at Adams County's 1.6% of Indiana AGI ($41,909.38), giving $670.55." -us,scenario_015,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four specified categories and left out Indiana county income tax. PolicyEngine includes that tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model read the four-city list as exhaustive and answered $0. PolicyEngine counts Indiana county income tax in local_income_tax, applying the default Adams County rate of 1.6% to Indiana AGI of $41,909.38, which gives $670.55." -us,scenario_015,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city and earnings taxes and concluded none apply. It missed Indiana county income tax, which PolicyEngine computes at Adams County's 1.6% on $41,909.38 of Indiana AGI ($42,909.38 federal AGI minus the $1,000 exemption), giving $670.55." -us,scenario_015,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"The model explicitly placed Indiana county taxes outside this output's scope. PolicyEngine includes them in local_income_tax at Adams County's 1.6% on Indiana AGI of $41,909.38, giving $670.55." -us,scenario_015,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model assumed PolicyEngine's local-income-tax output covers only NYC, Philadelphia, KC and St. Louis. PolicyEngine also includes Indiana county income tax and applies Adams County's 1.6% to $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model excluded Indiana county income tax from the output and took the missing county as meaning zero tax. PolicyEngine includes the county tax and defaults to Adams County, applying 1.6% to Indiana AGI of $41,909.38, which gives $670.55." -us,scenario_015,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model limited the output to the listed jurisdictions and answered $0, leaving out Indiana county income tax. PolicyEngine computes that tax at Adams County's 1.6% on $41,909.38 of Indiana AGI, giving $670.55." -us,scenario_015,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model checked only the four named city taxes and missed Indiana county income tax. PolicyEngine includes that tax in local_income_tax at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model read the prompt's list of four cities as the full scope of the output, so it left out the Indiana county income tax that PolicyEngine counts in local_income_tax. Using the default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55." +us,scenario_015,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana county taxes outside the output's scope because the prompt lists only NYC, Philadelphia, Kansas City and St. Louis. PolicyEngine's local_income_tax does include the Indiana county tax: 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model said Indiana has no local income tax. That is wrong: all 92 Indiana counties levy a county income tax on Indiana AGI. PolicyEngine applies the Adams County default rate of 1.6% to $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Based on the prompt's four-city list, the model ruled Indiana county income tax out of scope. PolicyEngine's local_income_tax includes it: 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed cities and never applied the Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model treated the output as covering only NYC, Philadelphia, Kansas City and St. Louis. PolicyEngine also puts Indiana county income tax in local_income_tax: 1.6% (Adams County default) on Indiana AGI of $41,909.38, or $670.55." +us,scenario_015,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model left Indiana county income tax out because the prompt lists only four city taxes. PolicyEngine reports that county tax in local_income_tax: 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model applied only the jurisdictions the prompt listed and missed the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is the Adams County default rate of 1.6% × Indiana AGI of $41,909.38 = $670.55." +us,scenario_015,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and never considered the Indiana county income tax, which PolicyEngine counts in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model read the four enumerated city taxes as the full scope of the output and left out the Indiana county income tax. PolicyEngine includes it at the Adams County default rate of 1.6% on $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana out because it is not among the four listed jurisdictions. PolicyEngine's local_income_tax still includes Indiana county income tax: 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model checked only the four enumerated city taxes and left out the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model treated the four listed cities as the only jurisdictions in scope and never applied the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model looked only at the four listed city earnings taxes and missed the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model said no local income tax applies in Indiana. That is wrong: every Indiana county levies a county income tax on Indiana AGI. PolicyEngine applies the Adams County default rate of 1.6% to $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model knew Indiana has county income taxes but excluded them because the prompt lists only four cities. PolicyEngine reports the county tax in local_income_tax: 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model gave no local tax because it relied on the prompt's four-city definition and never applied the Indiana county income tax. PolicyEngine includes that tax at 1.6% (Adams County default) on $41,909.38 Indiana AGI, giving $670.55." +us,scenario_015,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC, Philadelphia, Kansas City and St. Louis and left out the Indiana county income tax that PolicyEngine counts in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model checked only the four enumerated city taxes and missed the Indiana county income tax that PolicyEngine includes. That tax is 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model said there is no local income tax in this jurisdiction and never applied the Indiana county income tax. PolicyEngine includes that tax in local_income_tax: 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed city jurisdictions and left out the Indiana county income tax. PolicyEngine computes that tax at the Adams County default rate of 1.6% on $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model said only NYC, Philadelphia, Kansas City and St. Louis are modeled. PolicyEngine's local_income_tax also models Indiana county income tax: 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model applied only the four listed localities and never applied the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"The model treated the four specified city taxes as the full scope of the output and left out the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and missed the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because Indiana is not one of the listed city jurisdictions, so it never applied the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model set local tax to 0 because no locality was given. PolicyEngine falls back to a default county, Adams County, and applies its 1.6% county income tax to Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC, Philadelphia, Kansas City and St. Louis and left out the Indiana county income tax that PolicyEngine includes. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana out of the output's listed jurisdictions and never applied the Indiana county income tax. PolicyEngine computes that tax with no county input by using the Adams County default rate: 1.6% × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model said only the four city taxes are modeled separately. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model said Indiana has no listed local tax and left out the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed cities and never applied the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"The model checked only for the four enumerated city taxes and missed the Indiana county income tax that PolicyEngine counts in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is not in NYC, Philadelphia, Kansas City or St. Louis, so it never applied the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"The model treated the four listed cities as the full scope of the output and left out the Indiana county income tax. PolicyEngine includes that tax at the Adams County default rate of 1.6% on $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model said no Indiana local income tax applies and never applied the county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and missed the Indiana county income tax that PolicyEngine includes. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC, Philadelphia, Kansas City and St. Louis and left out the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model ruled out local tax because the household is not in any of the four listed cities, so it never applied the Indiana county income tax that PolicyEngine includes. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model treated the four listed categories as the full scope of the output and left out the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model checked only the four enumerated city taxes and never applied the Indiana county income tax that PolicyEngine counts in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed city taxes and left out the Indiana county income tax. PolicyEngine includes that tax at the Adams County default rate of 1.6% on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana county taxes outside the output's scope because the prompt lists only four cities. PolicyEngine reports that county tax in local_income_tax: 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model said PolicyEngine's local output covers only NYC, Philadelphia, Kansas City and St. Louis. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% (Adams County default) × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and missed the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55." +us,scenario_015,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model excluded Indiana county income tax as out of scope and because no county was given. PolicyEngine includes the county tax in local_income_tax and uses the Adams County default when no county is specified: 1.6% × $41,909.38 = $670.55." +us,scenario_015,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed jurisdictions and never applied the Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) on Indiana AGI of $41,909.38, giving $670.55." +us,scenario_015,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model checked only for the four enumerated city taxes and left out the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $41,909.38 = $670.55." us,scenario_015,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It subtracted the $2,080 health insurance premiums from wages as a pre-tax cafeteria-plan exclusion and applied 7.65% to $42,920. The FICA base here is the full $45,000 of gross wages, since the listed premiums are not a Section 125 salary reduction. The correct total is $2,790 + $652.50 = $3,442.50." us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It lowered FICA wages to $42,920 by treating the $2,080 health insurance premiums as a pre-tax payroll deduction. Employee Social Security and Medicare tax apply to the full $45,000 of gross wages, which gives $3,442.50 rather than $3,283." us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $2,080 employer-sponsored insurance premiums were pre-tax and cut FICA wages to $42,920. The 6.2% and 1.45% employee rates apply to the full $45,000 of gross wages, giving $2,790 + $652.50 = $3,442.50." @@ -1354,44 +1373,45 @@ us,scenario_016,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wron us,scenario_016,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_016,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_016,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its AGI, $16,100 standard deduction and brackets were all correct. It then also subtracted the $1,625 car-loan interest under the OBBBA deduction, but that deduction requires interest on a loan for a new, U.S.-assembled personal vehicle, which the facts don't establish. That understated taxable income by $1,625 and tax by $195." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 overtime-premium deduction it built from 50 weekly hours and a $1,625 car-loan interest deduction. The facts establish neither qualified overtime compensation nor a qualifying new-vehicle loan, so taxable income is $45,492, not $37,107." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It taxed the first $46,225 entirely at 12%, skipping the 10% bracket, and invented a 22% bracket. It then submitted $4,903, which doesn't match its own $5,716 calculation." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the health premiums from wages, but they are not listed as pre-tax payroll deductions and don't reduce AGI. It then gave a rough $4,700 instead of taxing $45,492 at 10% up to $12,400 and 12% above. Its own stated $50,000 taxable income would produce about $5,752, not $4,700." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It correctly computed $1,240 + 12% × $33,092 = $5,211. It then discarded that result for an unexplained 'rounded bracket' figure of $6,754, so the final answer contradicts its own correct calculation." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It set up the correct calculation, $1,240 + $3,971 on $45,492 of taxable income, which totals $5,211. It then reported $6,033, an addition error with no basis in the brackets it cited." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It assumed FLSA non-exempt status to claim a $6,760 overtime-premium deduction and assumed the car loan qualifies for a $1,625 interest deduction. The facts support neither, so taxable income is $45,492, not $37,107." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 10% ceiling of $11,925 instead of 2026's $16,100 and $12,400. It also claimed a $200 Saver's Credit using wrong income limits; a single filer's AGI of $61,592 is far above the single ceiling of about $40k, so the credit is $0." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a 10% bracket ceiling of $12,150 instead of $12,400, which gave $5,216. It then subtracted an unexplained ~$303 to reach $4,913, even though it acknowledged the Saver's Credit is phased out at this income and no other nonrefundable credit applies." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $9,000 premium twice as $18,000 and treated $1,500 of over-the-counter costs as deductible medical expenses. That inflated itemized deductions to $16,535, which it compared against a wrong $15,400 standard deduction. The correct medical-plus-SALT itemized total is well below the $16,100 standard deduction. It also used a wrong 10% ceiling of $12,075." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $7,700 standard deduction, a $4,950 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, giving a $16,100 standard deduction, no exemption and 10%/12% brackets. It also improperly subtracted $9,000 of health premiums from wages." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction and a $12,450 10% ceiling instead of the 2026 values of $16,100 and $12,400. That overstated taxable income by $450 and tax by about $53." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied a TCJA sunset: an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA kept the TCJA structure for 2026: a $16,100 standard deduction, no exemption, and 10% to $12,400 then 12%." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2024 $14,600 standard deduction instead of $16,100 and left out the $289 IRA deduction and the $50 of interest. Its $4,724 doesn't follow from any bracket calculation on its own $47,231 of taxable income." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used pre-TCJA 'sunset' law, with a combined deduction and personal exemption of $13,790 and 15%/25% brackets. The 2026 law has a $16,100 standard deduction, no personal exemption, and 10%/12% brackets up to $50,400." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, applying an $8,300 standard deduction, a $5,300 exemption and a 15% bracket instead of 2026's $16,100 deduction and 10%/12% brackets. It also subtracted $9,000 of health premiums from wages even though they aren't a listed pre-tax deduction." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no calculation. Its $5,304 is about $93 above the correct tax, which matches a standard deduction roughly $775 smaller than 2026's $16,100, meaning it used outdated deduction or bracket parameters." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $15,700 standard deduction and a $12,275 10% ceiling instead of 2026's $16,100 and $12,400. That overstated taxable income by $400." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a standard deduction plus personal exemption with 10%/15% brackets. OBBBA continues the $16,100 standard deduction, with no exemption and 10%/12% rates, for 2026." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 10% ceiling of $11,925 instead of the 2026 values of $16,100 and $12,400. That gave taxable income of $46,592 instead of $45,492." -us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,400 and the 10% ceiling at $12,260 instead of the published $16,100 and $12,400. That overstated taxable income by $700." -us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction (the 2025 pre-OBBBA figure) instead of 2026's $16,100, and a $12,300 10% ceiling instead of $12,400. Taxable income came out as $46,592 instead of $45,492." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It subtracted a nonrefundable Child Tax Credit for 'one qualifying child,' but the household is a single adult with no dependents. No credit applies against the $5,211.05 of regular tax." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed there was no remaining liability. A single filer with $61,592 AGI has $45,492 of taxable income after the $16,100 standard deduction and owes $5,211.05, with no nonrefundable credits to offset it." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 qualified-overtime deduction calculated from the 50-hour work week. The facts establish no qualified overtime compensation, so taxable income is $45,492, not $38,732." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its taxable income of $45,492 is correct, but its $5,220.54 implies it used the 2025 10% ceiling of $11,925 instead of 2026's $12,400. That overstated tax by $9.50." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It added an estimated $6,760 overtime-premium deduction (10 hours × $13 × 52 weeks) on top of the $16,100 standard deduction. The facts establish no qualified overtime compensation, so taxable income is $45,492." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It deducted $6,760 of qualified overtime, reducing taxable income to $38,732. No overtime deduction applies here, so the tax on $45,492 is $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It did no bracket calculation, mislabeled roughly $61k of AGI as taxable income, and gave a round $5,200. The correct calculation is $1,240 + 12% × $33,092 = $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a repealed $5,300 personal exemption and 10%/15% sunset brackets. It also built a $15,035 itemized total by counting $18,000 of premiums. The 2026 law gives a $16,100 standard deduction with no exemption, so taxable income is $45,492." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,250 10% ceiling instead of 2026's $16,100 and $12,400. That overstated taxable income by $700." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its taxable income of $45,492 is correct, but it ended the 10% bracket at $12,200 instead of 2026's $12,400. That pushed $200 more into the 12% bracket and gave $5,215.04." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $15,375 standard deduction instead of the 2026 single amount of $16,100. That overstated taxable income by $725, to $46,217." -us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It claimed about $15,381 of medical deductions by counting the $9,000 premium twice and including over-the-counter costs. That made itemized deductions of about $16,461 appear larger than the $16,100 standard deduction. The correct itemized total is far below $16,100, so the standard deduction applies and taxable income is $45,492." -us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for this variable, so there is no usable answer." -us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It gave $0 with no derivation. A single filer with $61,592 AGI and $45,492 of taxable income after the $16,100 standard deduction owes $5,211.05, with no nonrefundable credits to reduce it." -us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It left out the $289 deductible traditional IRA contribution, giving AGI of $61,881. It also used the $15,750 standard deduction (the 2025 figure) instead of 2026's $16,100, and a $12,550 10% ceiling instead of $12,400." -us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It understated taxable income by about $12,550 ($32,942 instead of $45,492) by subtracting retirement contributions beyond the 401(k) and IRA amounts already removed from AGI. It then claimed a $303 Saver's Credit even though a single filer's AGI of $61,592 is far above the credit's income ceiling." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model got AGI, the $16,100 standard deduction and the 2026 brackets right, but also subtracted $1,625 for the OBBBA car-loan interest deduction. That deduction requires a loan taken out after 2024 for a new, U.S.-assembled vehicle, and the facts show no such vehicle (household vehicles are worth $4,690). Without that subtraction, taxable income is $45,492.09 and tax is $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 overtime-premium deduction and a $1,625 car-loan interest deduction that don't apply. Listed wages of $68,005 are about $26 × 50 hrs × 52, which is straight-time pay with no qualified overtime premium, and no qualifying new vehicle loan is shown. Taxable income is $45,492.09, not $37,107." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction of $14,600 instead of the 2026 figure of $16,100. It also made up a bracket breakpoint (12% only up to $46,225) and then reported $4,903, which doesn't match its own $5,716. With the correct figures, taxable income is $45,492.09 and tax is 1,240 + 12% × 33,092.09 = $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted health premiums from wages, but no pre-tax premium exclusion is listed, so AGI is $61,592.09. It then guessed taxable income at about $50,000, and its $4,700 answer is even lower than the tax on that guess (about $5,752). The correct taxable income of $45,492.09 gives $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly worked out $1,240 + $3,971 = $5,211 on taxable income of $45,492. It then replaced that with an unsupported $6,754 as an 'adjustment with rounded brackets', throwing away the correct result." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly found taxable income of $45,492 and the bracket pieces $1,240 and $3,971, but reported a total of $6,033 instead of their sum, $5,211. This is an arithmetic error in the final step." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed the job was FLSA non-exempt and deducted a $6,760 overtime premium, plus $1,625 of car-loan interest. Wages of $68,005 equal straight-time pay of $26 × 50 × 52, so they contain no time-and-a-half premium, and the facts show no qualifying new U.S.-assembled vehicle loan. Neither deduction applies, so taxable income is $45,492.09." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 10% bracket ceiling of $11,925 instead of 2026's $16,100 and $12,400. It also took a $200 Saver's Credit using a made-up $64,000 single-filer limit. The real single limit is about $40,250, so at $61,592 AGI there is no Saver's Credit." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"After getting taxable income of $45,492 right, the model used a 10% bracket ceiling of $12,150 instead of $12,400. It then subtracted an unexplained $303 credit even though it had just said the Saver's Credit was phased out. At $61,592 AGI a single filer gets no Saver's Credit, so the answer is the full $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model counted the same $9,000 premium twice ($18,000) and included $1,500 of over-the-counter costs, which aren't deductible medical expenses. That pushed itemized deductions above a $15,400 standard deduction that is itself too low. Correctly measured, medical expenses after the 7.5% floor plus Arizona tax total about $6,000, well under $16,100, so the standard deduction applies." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and applied a $7,700 standard deduction, a personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no exemption, and 10%/12% brackets. It also wrongly cut AGI by $9,000 of health premiums; AGI is $61,592.09." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $15,650 standard deduction and a $12,450 10% bracket ceiling. The 2026 figures are $16,100 and $12,400, which give taxable income of $45,492.09 and tax of $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed TCJA expired and applied an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. Under OBBBA, 2026 has a $16,100 standard deduction, no personal exemption, and 10%/12% brackets on taxable income of $45,492.09." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction of $14,600 instead of $16,100 and left out the $289 IRA deduction and the $50 of interest. Its $4,724 is also below the tax on its own $47,231 taxable income (about $5,420 under 2026 brackets). The correct taxable income of $45,492.09 gives $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied pre-TCJA 2026 rules, with a combined $13,790 standard deduction and personal exemption plus higher brackets. OBBBA kept the TCJA structure: a $16,100 standard deduction with 10%/12% brackets, giving $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed TCJA expired (an $8,300 standard deduction, a $5,300 exemption and a 15% bracket) and also took $9,000 of health premiums out of AGI. Under current law, AGI is $61,592.09, the standard deduction is $16,100, and taxable income of $45,492.09 is taxed at 10%/12%." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. Its $5,304 matches a standard deduction of about $15,350–$15,400 instead of the 2026 figure of $16,100, which raises taxable income by roughly $750. The correct taxable income of $45,492.09 gives $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and a $12,275 10% bracket ceiling instead of 2026's $16,100 and $12,400. As a result it overstated taxable income at $45,892 instead of $45,492.09." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed TCJA expired and applied a personal exemption, a smaller standard deduction and a 15% bracket. OBBBA permanently set the 2026 single standard deduction at $16,100 with 10%/12% brackets, which gives $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 10% bracket ceiling of $11,925 instead of 2026's $16,100 and $12,400. That inflated taxable income to $46,592." +us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,400 standard deduction and a $12,260 10% bracket ceiling instead of 2026's $16,100 and $12,400. That overstated taxable income by $700." +us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction (the 2025 pre-OBBBA figure) and a $12,300 10% bracket ceiling instead of 2026's $16,100 and $12,400. That inflated taxable income to $46,592." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model subtracted a nonrefundable Child Tax Credit for 'one qualifying child', but the household is a single 53-year-old filer with no dependents. With no credits, tax on $45,492.09 of taxable income is $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model claimed there was no positive tax liability without computing anything. AGI of $61,592.09 minus the $16,100 standard deduction leaves $45,492.09 of taxable income, which produces $5,211.05 of tax, and no nonrefundable credit offsets it." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 qualified overtime deduction, but wages of $68,005 are straight-time pay ($26 × 50 × 52) and contain no time-and-a-half premium to deduct. Taxable income is $45,492.09, not $38,732." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly found taxable income of $45,492, but its $5,220.54 matches using the 2025 10% bracket ceiling of $11,925 instead of the 2026 ceiling of $12,400. That puts $475 more income in the 12% bracket." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 overtime-premium deduction (10 hrs × $13 × 52). The listed $68,005 in wages equals $26 × 50 × 52 straight time, so no qualified overtime premium was paid. Taxable income stays at $45,492.09." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 qualified overtime deduction that doesn't apply. The $68,005 in wages is straight-time pay ($26 × 50 × 52) with no overtime premium in it, so taxable income is $45,492.09 and tax is $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model mixed up AGI (about $61k) with taxable income and gave a rounded $5,200 instead of working through the brackets. The exact figure is 10% × $12,400 + 12% × $33,092.09 = $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied expired-TCJA rules (a $5,300 personal exemption and a 15% bracket). It also counted the same $9,000 premium twice to reach $18,000 of premiums in its itemized deductions. Under OBBBA, the $16,100 standard deduction is larger than the correctly measured itemized deductions, and the 10%/12% brackets apply." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction and a $12,250 10% bracket ceiling instead of 2026's $16,100 and $12,400. That overstated taxable income at $46,192." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model correctly found taxable income of $45,492, but used a 10% bracket ceiling of $12,200 instead of the 2026 ceiling of $12,400. That gives $5,215.04 instead of $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 standard deduction instead of the 2026 figure of $16,100. That produced taxable income of $46,217 instead of $45,492.09." +us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its $15,381 medical deduction counts the same $9,000 premium twice and includes non-deductible over-the-counter costs. Allowable medical expenses ($9,500 minus the 7.5% floor of $4,619) plus Arizona tax total about $6,000, far below the $16,100 standard deduction." +us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model gave $0 with no derivation, treating the filer as owing no tax. Taxable income of $45,492.09 after the $16,100 standard deduction produces $5,211.05 of tax, and no nonrefundable credits apply." +us,scenario_018,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 overtime-premium deduction and a $1,625 car-loan interest deduction. The $68,005 in wages is straight-time pay ($26 × 50 × 52) with no overtime premium, and no qualifying new U.S.-assembled vehicle loan is shown. Taxable income is $45,492.09, not $37,107." +us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model left out the $289 deductible IRA contribution, putting AGI at $61,881 instead of $61,592.09. It also used a $15,750 standard deduction (the 2025 figure) and a $12,550 10% bracket ceiling instead of 2026's $16,100 and $12,400." +us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its taxable income of $32,942 is about $12,550 below the correct $45,492.09, consistent with deducting retirement contributions a second time after they had already reduced AGI. It then took a $303 Saver's Credit, even though $61,592 AGI is above the single-filer limit of about $40,250." us,scenario_018,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $4,216.31 of Social Security tax and $986.07 of Medicare tax, which sum exactly to $5,202.38, then invented a $49.41 “rounding adjustment.” No such adjustment applies." us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $9,000 health-insurance premiums from FICA wages as though they were documented pre-tax employer-plan salary reductions. Applying 7.65% to the full $68,005 wage base yields $5,202.38." @@ -1437,84 +1457,86 @@ us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error, us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It assumed gross wages already excluded the traditional 401(k) and ignored the IRA deduction, which overstated AGI at $68,055 instead of $61,592.09. It also used a $14,600 standard deduction instead of $16,100." us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $14,508 standard deduction instead of $16,100. It then applied an invented 'rate structure' that taxes $45,486 at about 1.8%, when Arizona's flat rate is 2.5%." us,scenario_018,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"It counted only the $13,834 of real estate taxes as SALT and took the $16,100 standard deduction. It never added the Texas general sales tax deduction from the IRS optional table (~$3,256), which brings itemized SALT to $17,090 and makes itemizing better. Taxable income came out at $284,050 instead of $283,060, so its answer is about $347 too high." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction. It missed the optional-table sales tax deduction available to a Texas filer, which raises SALT to $17,090. With that, the filer itemizes, the $196 gift falls below the 0.5%-of-AGI floor, and taxable income is $283,060 rather than $283,854." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"It subtracted the $138,375 of unreimbursed employee business expenses above the line, but these remain nondeductible miscellaneous itemized deductions. It also used a $15,000 standard deduction. Its final $66,661 has no connection to its own $27,268 computation." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,other,False,"It admitted it never completed the computation and submitted 0 as a placeholder. The correct derivation (AGI $300,150, $17,090 itemized SALT, taxable income $283,060, plus NIIT) gives $68,056.71." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 of gross wages, which already include it. It capped SALT at $10,000 instead of the 2026 $40,400 cap and ignored the Texas sales tax deduction. It also omitted NIIT, and its reported $70,813 contradicts its own $82,743 computation." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"It made up about $23,855 of mortgage interest from the loan balance, although no interest paid was listed. It disallowed the alimony deduction and capped SALT at $10,000 rather than the 2026 $40,400 cap. Its reported $71,866 also does not match its own ~$64,291 total." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"It guessed itemized deductions of $30,000–40,000 without itemizing anything. The actual deductions are $13,834 of property tax plus ~$3,256 of table sales tax ($17,090). Its $72,700 is a figure with no derivation behind it." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"It compared only the $13,834 of property tax with the $16,100 standard deduction and left out the optional-table general sales tax deduction a Texas filer can claim. That deduction raises SALT to $17,090 and makes itemizing better. Taking the standard deduction plus the $196 non-itemizer charity left taxable income at $283,854 instead of $283,060." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction, capped SALT at $10,000, and skipped the Texas sales tax deduction. It projected 2025 brackets and a $15,400 standard deduction instead of the 2026 OBBBA values ($16,100), and it left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"It used 2025 brackets with a $15,000 standard deduction and capped SALT at $10,000, which made it miss that $17,090 of SALT (including table sales tax) beats the $16,100 deduction. It added the $900 Additional Medicare Tax, which is a payroll tax, to income tax. It then raised its own $70,467 to an arbitrary $76,500." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"It treated the $5,709 of alimony as nondeductible, putting AGI at $305,859 instead of $300,150. It also took the standard deduction instead of itemizing $17,090 of SALT (property tax plus Texas optional-table sales tax), so it taxed $289,759 instead of $283,060." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"It assumed TCJA had expired in 2026. On that basis it deducted the $138,375 of unreimbursed employee expenses above a 2% floor and claimed a personal exemption. OBBBA made both the miscellaneous-deduction suspension and the zero personal exemption permanent, so taxable income is $283,060, not ~$155,500." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"It applied pre-TCJA law: a 2%-floor deduction for employee expenses, a personal exemption, and 15/25/28% brackets, all of which OBBBA permanently replaced. It also subtracted the $8,389 of ESI premiums from the gross wages, which are already given." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"It took a $25,000 overtime deduction, but that is the joint cap. A single filer is capped at $12,500, and the cap phases out completely at $100 per $1,000 of MAGI over $150,000. It also skipped the alimony deduction, used a $15,350 standard deduction instead of itemizing $17,090 of SALT, and left out NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"It used pre-TCJA 2026 law, deducting employee business expenses above a 2% floor and claiming a personal exemption. OBBBA permanently eliminated both, so taxable income is $283,060, not $154,121." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"It modeled a TCJA sunset with miscellaneous itemized deductions, a personal exemption, and an AMT recapture, all of which OBBBA removed. It also subtracted ESI premiums from wages and skipped the alimony deduction. The correct path is to itemize $17,090 of SALT against AGI of $300,150." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"It applied a $10,000 SALT cap instead of the 2026 $40,400 cap, skipped the alimony deduction, and left out the Texas sales tax deduction. As a result it took the standard deduction instead of itemizing $17,090 against AGI of $300,150." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"It assumed TCJA had expired and allowed the unreimbursed employee expenses as a 2%-floor miscellaneous deduction, which then triggered AMT. OBBBA permanently disallows those expenses, so regular tax on $283,060 plus NIIT applies. It also subtracted ESI premiums from wages." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"It itemized $146,570, which includes the suspended employee business expenses, under an assumed TCJA sunset with AMT. It also subtracted ESI premiums from the gross wages. The correct itemized total is only the $17,090 of SALT." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"It skipped the alimony deduction, took the standard deduction instead of itemizing $17,090 of SALT, and left out NIIT. Its $50,568 is far below what the 2026 single brackets (35% above $256,225) produce on ~$290,000 of taxable income, so it misapplied the bracket schedule." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"It applied pre-TCJA rules: uncapped SALT, a 2%-floor deduction for the $138,375 of employee expenses, and personal exemptions. OBBBA permanently disallows employee expenses and exemptions. It also subtracted ESI premiums from wages." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"It assumed a TCJA sunset and deducted unreimbursed employee expenses above a 2% floor plus a personal exemption, both permanently eliminated by OBBBA. It also subtracted the $8,389 of ESI premiums from the gross wages." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"It skipped the alimony deduction and used a $15,000 standard deduction with 2025 brackets instead of the 2026 values. It missed the Texas sales tax deduction that makes itemizing $17,090 better, and it left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"It treated 2026 as a post-TCJA reversion year and deducted $132,372 of employee business expenses plus a personal exemption. OBBBA made the suspension of both permanent, so taxable income is $283,060, not ~$148,748." -us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction and counted only $13,834 of property tax plus $196 of charity as itemized deductions. It never added the Texas optional-table sales tax, which raises SALT to $17,090, so it wrongly took the $16,100 standard deduction on taxable income of $289,759." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"It claimed deductions wipe out all tax, relying on unlisted mortgage interest and the $138,375 of unreimbursed employee expenses. Those expenses are permanently nondeductible, and no interest paid was given. Itemized deductions total only $17,090, which leaves $283,060 taxable." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"Its ~$31,472 matches a pre-TCJA calculation that deducts the $138,375 of employee business expenses as a miscellaneous itemized deduction. OBBBA permanently disallows that deduction, so the correct taxable income is $283,060." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"It took the $16,100 standard deduction without counting the Texas optional-table general sales tax deduction (~$3,256). With it, SALT is $17,090 and itemizing is better. Taxable income came out $990 too high ($284,050 instead of $283,060)." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"It subtracted the $8,389 of employer-sponsored insurance premiums from the given gross wages and left out the $222.64 NIIT. It also took the standard deduction instead of itemizing $17,090 of SALT, which includes Texas table sales tax." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 non-itemizer charity deduction, missing that property tax plus Texas optional-table sales tax ($17,090) makes itemizing better. It also left out the $222.64 net investment income tax that belongs in the total." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"It skipped the $5,709 alimony deduction and took the standard deduction instead of itemizing $17,090 of SALT, which includes the Texas sales tax table amount. It also left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"It used the $16,100 standard deduction plus the $196 non-itemizer charitable deduction instead of itemizing $17,090 of SALT (property tax plus Texas optional-table sales tax). It also left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"It skipped the $5,709 above-the-line alimony deduction and took the standard deduction instead of itemizing $17,090 of SALT, which includes Texas general sales tax. It therefore taxed about $289,759 instead of $283,060." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"It started from $305,859 without deducting the $5,709 of alimony and applied the $16,100 standard deduction. It missed that property tax plus the Texas optional-table sales tax ($17,090) makes itemizing better, so it taxed $289,759 instead of $283,060." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"It chose the standard deduction plus the $196 non-itemizer charity deduction instead of itemizing $17,090 of SALT, which includes the Texas sales tax table amount. It also left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"It gave a rounded $65,000 guess based on the standard deduction, with no computation. It skipped the alimony deduction, the itemized SALT with Texas sales tax, the 2026 brackets, and NIIT, which together yield $68,056.71." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"It applied pre-TCJA 2026 law, deducting $132,372 of employee business expenses above a 2% floor and claiming a personal exemption. OBBBA permanently disallows both, so taxable income is $283,060, not $148,321." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"It assumed a TCJA sunset in which employee expenses and uncapped real estate taxes are deductible and then added back under AMT. OBBBA keeps employee expenses permanently nondeductible, so regular tax on $283,060 applies. It also skipped the alimony deduction." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"It took the $16,100 standard deduction without adding the Texas optional-table general sales tax to the $13,834 of property tax. That combined $17,090 makes itemizing better. It also left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,"It used projected pre-TCJA 10/15/25/28% brackets and deducted the $138,375 of employee expenses above a 2% floor. OBBBA permanently eliminated that deduction and kept the TCJA brackets. It also subtracted ESI premiums from the gross wages." -us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"It subtracted the $8,389 of ESI premiums from the gross wages, treated the $196 of charity as an above-the-line deduction, and skipped the $5,709 alimony deduction. It then took the standard deduction instead of itemizing $17,090 of SALT, which includes Texas table sales tax." -us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so the output is missing." -us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"It compared only the $13,834 of property tax with the $16,100 standard deduction and left out the Texas optional-table sales tax, which raises SALT to $17,090 and makes itemizing better. Taking the standard deduction plus the $196 non-itemizer charity left taxable income at $283,854 instead of $283,060." -us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"It submitted 0 with no reasoning, which means it computed no tax at all. The correct calculation is AGI of $300,150 less $17,090 of itemized SALT, giving $283,060 taxable, with $67,834 of tax plus $222.64 of NIIT, for $68,056.71." -us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction and counted only property tax plus charity ($14,030) as itemized deductions, never adding the Texas optional-table sales tax that raises SALT to $17,090. It also left out the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"It added the $42,857 overtime premium on top of the $300,000 of gross wages, which already include it. It capped SALT at $10,000 instead of $40,400 and used a $15,750 standard deduction. Its reported $68,513.50 contradicts its own $85,241.65 computation." -us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"It used married-filing-jointly brackets for a single filer, took a QBI deduction with no business income, and subtracted a child tax credit for a household with no children." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"The standard deduction is the household's choice under c_irs_sales_tax_2025, whose $1,595 IRS 2025 sales tax amount leaves SALT below $16,100. It omitted the $196 non-itemizer charitable deduction, giving taxable income of $284,050 instead of $283,854.19 and tax of $68,404 instead of $68,334.73 with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"Used the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, as appropriate under c_irs_sales_tax_2025 because the IRS 2025 sales tax amount leaves SALT below the standard deduction. Its $68,334.59 is $0.14 below the conventions' $68,334.73 with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"Subtracted the $138,375 of unreimbursed employee business expenses above the line, although they aren't deductible. It also used a $15,000 standard deduction instead of the 2026 $16,100 standard deduction plus the $196 non-itemizer charitable deduction appropriate under c_irs_sales_tax_2025. Its final $66,661 doesn't follow from its own $27,268 bracket calculation." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,other,False,"Never computed the liability and submitted $0, even though it noted the $300,000 of wages produces a large tax. Under c_irs_sales_tax_2025, $300,150.19 AGI minus the $16,100 standard deduction and $196 non-itemizer charitable deduction gives $283,854.19 of taxable income and $68,334.73 with NIIT, or the exclusion record's $68,112.09 without it." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"Added the $42,857 FLSA overtime premium on top of the $300,000 of gross wages, even though it is already included in them. It applied a $10,000 SALT cap instead of the 2026 $40,400 cap and missed the Texas general sales tax deduction. Its submitted $70,813 doesn't match its own $82,743 calculation, and it left out NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"Imputed $23,855 of mortgage interest from the loan balance, although unlisted interest is $0. It also denied the $5,709 alimony deduction, capped SALT at $10,000 instead of $40,400, and missed the sales tax deduction. Its submitted $71,866 doesn't match its own ~$64,291 arithmetic." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"Guessed itemized deductions of $30,000–$40,000 with no listed basis; c_irs_sales_tax_2025 leaves SALT below the $16,100 standard deduction, which combines with the $196 non-itemizer charitable deduction. Its final $72,700 estimate is unexplained; the convention result is $68,334.73 with NIIT, or $68,112.09 without it." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"Taking the $16,100 standard deduction plus the $196 non-itemizer charitable deduction agrees with c_irs_sales_tax_2025, whose IRS 2025 sales tax amount leaves SALT below the standard deduction. Its $68,334.59 is $0.14 below the conventions' $68,334.73 with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"Denied the $5,709 alimony deduction and capped SALT at $10,000 instead of the 2026 $40,400 cap. Under c_irs_sales_tax_2025, the IRS 2025 sales tax amount leaves SALT below the standard deduction; it used projected brackets and a $15,400 standard deduction instead of the 2026 parameters. It also left out the $222.64 of NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"Used 2025 brackets and a $15,000 standard deduction instead of the 2026 figures, and missed the Texas sales tax itemized deduction. It added $900 of Additional Medicare Tax, which is a payroll tax and not part of income tax. Then it arbitrarily inflated its own ~$70,467 total to $76,500." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"Denied the $5,709 above-the-line alimony deduction, giving AGI of $305,859 instead of $300,150.19. It also omitted the $196 non-itemizer charitable deduction from its standard deduction, giving taxable income of $289,759 instead of $283,854.19 under c_irs_sales_tax_2025." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"Applied pre-TCJA rules: personal exemptions and $138,375 of employee business expenses as miscellaneous itemized deductions above a 2% floor. Both remain permanently repealed in 2026 under OBBBA. This cut taxable income to about $155,500 instead of $283,854.19 under c_irs_sales_tax_2025." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"Applied pre-TCJA rules (personal exemption, miscellaneous itemized deduction of employee business expenses, 15/25/28% brackets), which OBBBA permanently replaced for 2026. It also subtracted the $8,389 of ESI premiums from the stated gross wages and left out both the alimony deduction and NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"Claimed a $25,000 overtime deduction, but the single-filer deduction is capped at $12,500 and phases out by $100 per $1,000 of MAGI over $150,000, so it is zero at $300,150 AGI. It also denied the alimony deduction, used a $15,350 standard deduction instead of the $16,100 standard deduction plus the $196 non-itemizer charitable deduction appropriate under c_irs_sales_tax_2025, and left out NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"Used pre-TCJA law: a personal exemption, unreimbursed employee expenses as miscellaneous itemized deductions above a 2% floor, and 10/15/25/28% brackets. All of these stay repealed in 2026 under OBBBA. This understated taxable income by about $129,000 and left out NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"Assumed the TCJA sunset and allowed employee business expenses plus a personal exemption, then replaced the regular tax with an AMT estimate. It also subtracted $8,389 of ESI premiums from the stated gross wages and skipped the alimony deduction. The correct regular-tax path under c_irs_sales_tax_2025 uses the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, giving $283,854.19 of taxable income." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"Denied the alimony deduction, giving AGI of $305,859, and capped SALT at $10,000 instead of $40,400. Under c_irs_sales_tax_2025, SALT remains below the standard deduction, so the household uses the $16,100 standard deduction plus the $196 non-itemizer charitable deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"Assumed the TCJA provisions expire in 2026 and allowed unreimbursed employee expenses as miscellaneous itemized deductions, then calculated an AMT. OBBBA made the suspension permanent. It also subtracted ESI premiums from the stated gross wages." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"Counted $146,570 of itemized deductions, including the $138,375 of unreimbursed employee expenses, which are permanently nondeductible under OBBBA. It also subtracted $8,389 of ESI premiums from gross wages and replaced the regular tax with an AMT estimate. Under c_irs_sales_tax_2025, SALT remains below the $16,100 standard deduction, so the household uses that deduction plus the $196 non-itemizer charitable deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"Left out the alimony deduction and the Texas sales tax itemization. Its $50,568 is about $18,000 below what the 2026 single brackets produce for roughly $290,000 of taxable income, so the bracket arithmetic is wrong." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"Applied pre-TCJA law, with personal exemptions, uncapped SALT and miscellaneous employee business expenses above a 2% floor, all of which OBBBA keeps repealed for 2026. It also subtracted ESI premiums from the stated gross wages." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"Assumed the TCJA sunset and itemized $138,375 of unreimbursed employee expenses above a 2% floor, plus a personal exemption, although both stay repealed in 2026. It also subtracted $8,389 of ESI premiums from the stated gross wages and skipped the alimony deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"Used the 2025 $15,000 standard deduction and brackets instead of the 2026 figures and denied the alimony deduction. Under c_irs_sales_tax_2025, the household uses the $16,100 standard deduction plus the $196 non-itemizer charitable deduction; it left out the $222.64 NIIT included in the frozen reference." +us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"Assumed the TCJA expires in 2026 and deducted $132,372 of unreimbursed employee expenses as miscellaneous itemized deductions plus a personal exemption, although OBBBA made both repeals permanent. It also used pre-TCJA brackets and left out NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"Denied the $5,709 above-the-line alimony deduction, giving AGI of $305,859. Its standard deduction is appropriate under c_irs_sales_tax_2025, whose IRS 2025 table leaves SALT below $16,100; it omitted the $196 non-itemizer charitable deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"Concluded deductions wipe out the tax by imputing mortgage interest (unlisted, so $0) and allowing the $138,375 of unreimbursed employee expenses, which are nondeductible. Under c_irs_sales_tax_2025, SALT stays below the $16,100 standard deduction, and that deduction plus $196 of non-itemizer charity leaves $283,854.19 of taxable income." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"Its ~$31,472 answer matches a pre-TCJA calculation that deducts the unreimbursed employee expenses as miscellaneous itemized deductions. Those remain permanently disallowed in 2026, so taxable income is about $130,000 too low." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"Took the $16,100 standard deduction, which is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below it, but omitted the $196 non-itemizer charitable deduction; the applicable taxable income is $283,854.19." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"Subtracted the $8,389 of ESI premiums from the stated $300,000 gross wages, which understates income. Its standard deduction is appropriate under c_irs_sales_tax_2025, whose IRS 2025 table leaves SALT below $16,100; it omitted the $196 non-itemizer charitable deduction and, compared with the frozen reference, the $222.64 of NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"Used the standard deduction plus the $196 non-itemizer charitable deduction, as appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100. Its $68,112.00 is $0.09 below the exclusion record's $68,112.09 without NIIT; adjudication leaves NIIT inclusion ambiguous." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"Its $70,179 matches $289,759 of taxable income. That means it denied the $5,709 alimony deduction and omitted the $196 non-itemizer charitable deduction; its standard deduction is appropriate under c_irs_sales_tax_2025. Compared with the frozen reference, it also left out the $222.64 of NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"Took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, as appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100; its $68,111.95 is $0.14 below the exclusion record's $68,112.09 without NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"Its $70,401 matches $289,759 of taxable income plus NIIT. That means it denied the $5,709 alimony deduction and omitted the $196 non-itemizer charitable deduction; its standard deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"Started from $305,859 of gross income with no $5,709 alimony deduction and took the $16,100 standard deduction. Its standard deduction is appropriate under c_irs_sales_tax_2025, but it omitted the $196 non-itemizer charitable deduction; with alimony and charity, taxable income is $283,854.19 rather than its $289,759." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"Used the standard deduction plus the $196 non-itemizer charitable deduction ($283,854 of taxable income), as appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100; its $68,111.95 is $0.14 below the exclusion record's $68,112.09 without NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"Gave a rounded $65,000 guess with no computation. Under c_irs_sales_tax_2025, the path uses $300,150.19 of AGI after alimony, the $16,100 standard deduction plus $196 of non-itemizer charity, and $68,112.09 of regular tax, or $68,334.73 with NIIT; adjudication leaves NIIT inclusion ambiguous." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"Applied pre-TCJA rules, deducting $132,372 of employee business expenses as miscellaneous itemized deductions and claiming a personal exemption. OBBBA keeps both repealed in 2026, so taxable income came out at $148,321 instead of $283,854.19 under c_irs_sales_tax_2025, which takes the standard deduction plus $196 of non-itemizer charity." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"Assumed the TCJA sunset and allowed uncapped SALT and miscellaneous employee expenses, then estimated AMT. It also denied the alimony deduction. Under 2026 law regular tax applies with no AMT, on $283,854.19 of taxable income under c_irs_sales_tax_2025, which takes the standard deduction plus $196 of non-itemizer charity." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"Took the $16,100 standard deduction, which is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below it, but omitted the $196 non-itemizer charitable deduction. It also left out the $222.64 of NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,"Applied pre-TCJA rules: no SALT cap, miscellaneous deduction of the $138,375 of employee expenses, and 15/25/28% brackets. All of these are superseded in 2026. It also subtracted ESI premiums from the stated gross wages and skipped the alimony deduction and NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"Subtracted $8,389 of ESI premiums from the stated gross wages, denied the $5,709 alimony deduction, and treated the $196 charitable gift as an above-the-line adjustment. Its standard deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 sales tax table leaves SALT below $16,100." +us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Didn't return a value or any reasoning for federal_income_tax_before_refundable_credits. +us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"Its standard deduction plus the $196 non-itemizer charitable deduction is the household's choice under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100; its $68,334.59 is $0.14 below the convention result of $68,334.73 with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"Submitted $0 with no computation, ignoring the tax on $300,000 of wages. Under c_irs_sales_tax_2025, taxable income is $283,854.19 after alimony, the $16,100 standard deduction and $196 of non-itemizer charity, giving $68,112.09 of regular tax or $68,334.73 with NIIT; $68,056.71 is the excluded frozen reference." +us,scenario_020,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,other,False,"Subtracted the $8,389 of ESI premiums from the stated gross wages and left out the $5,709 alimony deduction. Its standard deduction plus the non-itemizer charitable deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100, and it left out NIIT compared with the frozen reference." +us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"Declared alimony nondeductible, leaving AGI at $305,859, and counted only $13,834 of property tax as SALT. Its standard deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100, but it omitted the $196 non-itemizer charitable deduction. It also left out the $222.64 of NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"Added the $42,857 FLSA overtime premium on top of the $300,000 of gross wages, even though it is already included, and capped SALT at $10,000 instead of $40,400. It also used 2025 brackets and a $15,750 standard deduction. Its submitted $68,513.50 contradicts its own $85,241.65 calculation." +us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"Used married-filing-jointly brackets for a single filer, applied a QBI deduction with no business income, and subtracted a child tax credit with no children." us,scenario_020,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_020,payroll_tax,claude-fable-5,llm_error,other,False,"It first reached the correct $16,689 ($11,439 + $4,350 + $900). It then wrongly subtracted the $8,389 ESI premiums from Medicare wages to get $16,491.86. Finally it submitted $15,200, which matches neither of its own totals." -us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500. It also added the $42,857 FLSA overtime premium on top of the $300,000 gross wages, which already include overtime, and subtracted the ESI premiums from FICA wages. Its submitted $23,119.50 then contradicts its own final sum of $16,513.19." -us,scenario_020,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"It listed the right components (capped Social Security, Medicare and Additional Medicare Tax) but never calculated them and submitted $0. The correct calculation gives $11,439 + $4,350 + $900 = $16,689." -us,scenario_020,payroll_tax,claude-opus-4.7,llm_error,other,False,"It calculated the correct $16,689 using the $184,500 base, then tried other wage bases ($176,100 and $183,600). It submitted $13,283.40, which does not match any of its own calculations and is below even its lowest alternative of $16,168.20." -us,scenario_020,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500, which understates Social Security tax by $520.80 and gives $16,168.20. It then submitted $6,219, which does not match its own total either." -us,scenario_020,payroll_tax,claude-opus-5,llm_error,other,False,"It calculated the correct $16,689 from the $184,500 base, $4,350 Medicare and $900 Additional Medicare Tax. It then cut this to an arbitrary $15,866 to account for 'base uncertainty' instead of submitting its own calculation." -us,scenario_020,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It carried the 2025 wage base of $176,100 into 2026 instead of using the 2026 base of $184,500. That gives Social Security tax of $10,918.20 instead of $11,439, a total that is $520.80 too low." -us,scenario_020,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It first calculated the correct $16,689 with the $184,500 base, then switched to a made-up base of $182,600. That gives Social Security tax of $11,321.20 and a submitted total of $16,571.20." -us,scenario_020,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a guessed 2026 wage base of $183,700 instead of $184,500. That gives Social Security tax of $11,389 instead of $11,439, so its total is $50 short." -us,scenario_020,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a wage base of $184,200 instead of the 2026 base of $184,500. That gives Social Security tax of $11,420.40 instead of $11,439." -us,scenario_020,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It used the 2025 wage base of $176,100 instead of $184,500. It also subtracted the $8,389 ESI premiums from Medicare wages, taxing $291,611 instead of $300,000, which lowered both Medicare tax ($4,228.36) and Additional Medicare Tax ($824.50)." -us,scenario_020,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"With $4,350 Medicare and $900 Additional Medicare Tax, its $15,516 total implies Social Security tax of about $10,266. That equals a wage base of about $165,600, far below the 2026 base of $184,500, so it applied an outdated cap." -us,scenario_020,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It explicitly reduced Medicare wages by the ESI premiums to $291,611, giving $4,228.36 Medicare tax and $824.50 Additional Medicare Tax. Its $16,436 total also implies a wage base of $183,600 instead of $184,500. Both choices lower the total." -us,scenario_020,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted the $8,389 ESI premiums from Medicare wages ($291,611), even though the full $300,000 is subject to Medicare tax. It also used a wage base of about $177,300, giving Social Security tax of about $10,993 instead of $11,439." -us,scenario_020,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"After $4,350 Medicare and $900 Additional Medicare Tax, its $22,864 total implies Social Security tax of about $17,614. That is consistent with applying 6.2% to nearly all wages without capping them at the $184,500 wage base, where the maximum tax is $11,439." -us,scenario_020,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It used a wage base of $181,800 instead of $184,500, giving Social Security tax of $11,271.60. It also subtracted the ESI premiums from Medicare wages ($291,611), which understates Medicare tax and Additional Medicare Tax by $121.64 and $75.50." -us,scenario_020,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $16,324.46 total equals $11,271.60 + $4,228.36 + $824.50. That means it used a $181,800 wage base instead of $184,500 and wrongly subtracted the $8,389 ESI premiums from Medicare wages." -us,scenario_020,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That gives Social Security tax of $10,918.20 instead of $11,439." -us,scenario_020,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"It used a rounded estimate of $180,000 for the 2026 wage base instead of $184,500. That gives Social Security tax of $11,160 instead of $11,439." -us,scenario_020,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"It used a wage base of $181,800 instead of the 2026 base of $184,500. That gives Social Security tax of $11,271.60 instead of $11,439, so its total is $167.40 short." -us,scenario_020,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $18,348 total is $1,659 above the maximum possible with a capped Social Security wage base, Medicare on $300,000, and Additional Medicare Tax on $100,000. That means it overstated the Social Security component beyond the 6.2% × $184,500 = $11,439 cap." -us,scenario_020,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It wrongly said no Additional Medicare Tax applied, even though wages exceed the $200,000 single threshold by $100,000 ($900 owed). Its $11,689 is also $4,100 below the correct $15,789 for Social Security plus regular Medicare." -us,scenario_020,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of $184,500. It then added up its own components wrongly: $10,918.20 + $4,350 + $900 equals $16,168.20, not $10,578." -us,scenario_020,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"It used an estimated 2026 wage base of $183,144 instead of the published $184,500. That gives Social Security tax of $11,355 instead of $11,439." -us,scenario_020,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It reduced FICA wages to $291,611 by subtracting the ESI premiums, which understates Medicare tax and Additional Medicare Tax. It also used a $184,000 wage base instead of $184,500, giving Social Security tax of $11,408 instead of $11,439." -us,scenario_020,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It used the correct $184,500 base ($11,439), but it subtracted the $8,389 ESI premiums from Medicare wages. That gave $4,228 Medicare and $825 Additional Medicare Tax instead of $4,350 and $900 on the full $300,000." -us,scenario_020,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score against the $16,689 total." -us,scenario_020,payroll_tax,minimax-m3,llm_error,missing_output,False,"It submitted $0 with no calculation. It ignored the Social Security, Medicare and Additional Medicare Tax owed on $300,000 of wages, which total $11,439 + $4,350 + $900 = $16,689." -us,scenario_020,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 gross wages, even though gross wages already include overtime, which inflated Medicare and Additional Medicare Tax. It also used the 2025 base of $176,100 instead of $184,500. Its submitted $19,436.31 does not match its own sum of $17,175.34." -us,scenario_020,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It said overtime premiums are included in the base, which is consistent with adding the $42,857 FLSA premium on top of gross wages that already include it. Even that double-count, with the correct cap, gives only about $17,696. Its $20,303 total therefore also overstates Social Security tax beyond the $11,439 maximum." +us,scenario_020,payroll_tax,claude-fable-5,llm_error,other,False,"It first computed the correct $16,689 using the $184,500 wage base. It then cut Medicare wages to $291,611 by treating the $8,389 ESI premium as a pre-tax deduction, which gave $16,491.86. The $15,200 it finally submitted matches neither figure." +us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It used the 2024 wage base of $168,600 instead of the 2026 base of $184,500. It also added the $42,857 overtime premium on top of gross wages that already include it, and subtracted the ESI premium from FICA wages. The $23,119.50 it submitted matches none of its own totals ($15,703.20, $16,710.34, $16,513.19)." +us,scenario_020,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It never computed the tax and submitted $0, even though $300,000 of wages produces $11,439 Social Security, $4,350 Medicare, and $900 Additional Medicare Tax, for $16,689 in total." +us,scenario_020,payroll_tax,claude-opus-4.7,llm_error,other,False,"Its first calculation was the correct $16,689 ($184,500 base, $4,350 Medicare, $900 Additional Medicare Tax). It then tried other wage bases and submitted $13,283.40, which matches none of its calculations." +us,scenario_020,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500, which gave $16,168.20. It then submitted $6,219, a number unrelated to that total." +us,scenario_020,payroll_tax,claude-opus-5,llm_error,other,False,"It computed the correct $16,689 from the $184,500 wage base, $4,350 Medicare, and $900 Additional Medicare Tax. It then cut that to $15,866 for 'base uncertainty' with no computational basis." +us,scenario_020,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It carried the 2025 wage base of $176,100 forward to 2026 instead of using $184,500. That understated Social Security tax by $520.80 and gave $16,168.20." +us,scenario_020,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct $16,689 with the $184,500 base, then 'rechecked' and switched to a wrong 2026 base of $182,600. That lowered Social Security tax to $11,321.20 and the total to $16,571.20." +us,scenario_020,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a guessed 2026 wage base of $183,700 instead of $184,500. That understated Social Security tax by $49.60 ($11,389 vs $11,439) and gave $16,639." +us,scenario_020,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a wage base of $184,200 instead of the 2026 base of $184,500. That made Social Security tax $11,420.40 instead of $11,439 and the total $16,670.40." +us,scenario_020,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It used the 2025 wage base of $176,100 instead of $184,500. It also cut Medicare wages to $291,611 by subtracting the $8,389 ESI premium, but those premiums do not reduce FICA wages. The result was Medicare of $4,228.36 and Additional Medicare Tax of $824.50 instead of $4,350 and $900." +us,scenario_020,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"With $4,350 Medicare and $900 Additional Medicare Tax, its $15,516 total implies about $10,266 of Social Security tax. That matches an outdated wage cap of about $165,600 instead of the 2026 base of $184,500, which yields $11,439." +us,scenario_020,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut Medicare wages by the ESI premium ('gross wages minus ESI'), but those premiums do not reduce FICA wages. Its $16,436 also matches a $183,600 wage base ($11,383) rather than $184,500. The correct figures are $11,439 + $4,350 + $900." +us,scenario_020,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It used a wage cap of about $177,300 (Social Security tax of about $10,993) instead of $184,500. It also cut Medicare wages to $291,611 by subtracting the ESI premium, which does not reduce FICA wages. Both errors understate the correct $11,439 + $4,350 + $900." +us,scenario_020,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $22,864 is consistent with applying 6.2% to roughly all $300,000 of wages (about $18,600), so it effectively ignored the $184,500 cap. With the cap, Social Security tax is $11,439 and the total is $16,689." +us,scenario_020,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It used a 2026 wage base of $181,800 instead of $184,500. It also cut Medicare wages to $291,611 by subtracting the $8,389 ESI premium, which does not reduce FICA wages. That gave $11,271.60 + $4,228.36 + $824.50 instead of $11,439 + $4,350 + $900." +us,scenario_020,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $16,324.46 equals $11,271.60 (a $181,800 wage base) plus $4,228.36 plus $824.50. That means it used the wrong cap and also cut Medicare wages by the ESI premium, which does not reduce FICA wages." +us,scenario_020,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 wage base of $176,100 ($10,918.20) instead of the 2026 base of $184,500 ($11,439). That gave $16,168.20 instead of $16,689." +us,scenario_020,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"It used a rounded guess of $180,000 for the 2026 wage base instead of $184,500. That made Social Security tax $11,160 instead of $11,439 and the total $16,410." +us,scenario_020,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"It used a 2026 wage base of $181,800 instead of $184,500. That understated Social Security tax by $167.40 and gave $16,521.60." +us,scenario_020,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $18,348 is $1,659 above the correct total. With the stated $4,350 Medicare and $900 Additional Medicare Tax, it implies Social Security tax of $13,098, which exceeds the $11,439 maximum set by the $184,500 cap." +us,scenario_020,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It said no Additional Medicare Tax was triggered, but $300,000 of wages exceeds the $200,000 single threshold and owes $900. Its $11,689 is also $5,000 below the correct $16,689, so its Social Security and Medicare components were understated as well." +us,scenario_020,payroll_tax,grok-4.3,llm_error,other,False,"Its stated components ($176,100 × 6.2% + $4,350 + $900) add to $16,168.20, but it reported $10,578, which is an arithmetic error. Separately, $176,100 is the 2025 cap, not the 2026 cap of $184,500." +us,scenario_020,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"It used an estimated wage base of $183,144 instead of $184,500. That made Social Security tax $11,355 instead of $11,439 and the total $16,605." +us,scenario_020,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It cut FICA wages to $291,611 by subtracting the ESI premium, so Medicare came out at $4,228 and Additional Medicare Tax at $825 instead of $4,350 and $900. It also used a $184,000 base instead of $184,500." +us,scenario_020,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It used the correct $184,500 base ($11,439) but subtracted the $8,389 ESI premium from Medicare wages. ESI premiums do not reduce FICA wages, so Medicare should be on the full $300,000 ($4,350) and Additional Medicare Tax should be $900, not $4,228 and $825." +us,scenario_020,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_020,payroll_tax,minimax-m3,llm_error,other,False,"It submitted $0 with no reasoning, even though $300,000 of wages owes $11,439 Social Security, $4,350 Medicare, and $900 Additional Medicare Tax, for $16,689 in total." +us,scenario_020,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"It treated the $8,389 ESI premium as a pre-tax deduction that cut FICA wages to $291,611, which gave Medicare of $4,228.36 and Additional Medicare Tax of $824.50. ESI premiums do not reduce FICA wages, so the full $300,000 gives $4,350 and $900." +us,scenario_020,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It added the $42,857 FLSA overtime premium on top of $300,000 gross wages that already include overtime, which inflated Medicare and Additional Medicare Tax. It also used the 2025 base of $176,100. On top of that, the $19,436.31 it submitted does not match its own $17,175.34 total." +us,scenario_020,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It treated overtime premiums as wages on top of the stated gross. Its $20,303 is far above the correct total, which caps Social Security at $184,500 ($11,439) and taxes $300,000 for Medicare ($4,350) plus $900 Additional Medicare Tax." us,scenario_020,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model correctly identified that Texas has no state individual income tax but then inserted an unexplained $2,380 nonrefundable state-tax amount. No state or local jurisdictional rule applies to create that liability for this Texas household, so the state-income-tax computation yields $0." us,scenario_021,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_021,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model assumed that being 85, disabled and on Medicare opens a Medicaid pathway automatically, and it never ran an income test. Missouri's ABD pathway applies an 85% FPL limit. The head's full Social Security plus pension (about $27,510 after the $20 disregard, or 172% of FPL) fails that limit, and at 85 the head is outside the MAGI adult group (ages 19–64)." @@ -1530,187 +1552,191 @@ us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for the requested output and therefore failed the required submission contract. us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,508 taxable pension as Missouri taxable income, ignored the applicable deductions and 2026 taxable-income threshold, and applied an unsupported flat 1.9% rate directly to that pension. Missouri taxable income is reduced to zero before the rate calculation, yielding $0 of tax before refundable credits." us,scenario_021,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It got single status, the 2026 single brackets, the $3,951 phased senior deduction and the $910 QBI deduction right. However, it itemized only $8,829 of property tax plus $15,393 of charity. It left out the ~$6,059 of California income tax that belongs in SALT and skipped the 0.5%-of-AGI charitable floor, so it reached taxable income of $80,066 instead of $74,552.59 and overstated tax by about $1,213." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse with joint brackets and the $32,200 joint standard deduction. That status requires a dependent child, so this head files single, where itemized deductions of $29,735.52 beat the $18,150 standard deduction. It also took the full $6,000 senior deduction instead of phasing it down by 6% of MAGI above $75,000 to $3,951." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used head-of-household status, which needs a qualifying person. It also subtracted the $8,351 of unreimbursed employee expenses above the line, though they are not deductible at all, and counted 100% of Social Security instead of the 85% maximum. Its reported $11,316 does not match its own $10,564 computation; correct single filing with $34,596.57 of total deductions gives $11,113.57." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed taxable Social Security was about $0 and that the senior standard deduction wiped out taxable income. In fact, $91,524 of other income pushes provisional income far over the thresholds, so 85% of benefits ($17,624.75) is taxable. AGI is $109,149.16, and after $34,596.57 of deductions, $74,552.59 is taxable, producing $11,113.57 rather than $0." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse joint brackets and a $33,950 standard deduction, but without a dependent child the head files single. It also left out the OBBBA senior deduction ($3,951 after phase-down) and the California income tax in SALT. Under single status, $29,735.52 of itemized deductions plus QBI and the senior deduction give taxable income of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse with joint brackets despite having no dependent child. It also made up ~$12,771 of mortgage interest from the loan balance when no interest paid is listed, and took the unphased $6,000 senior deduction instead of $3,951. Its reported $6,919 does not follow from its own ~$7,350 estimate." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It took the full $6,000 senior deduction, but MAGI of $109,149 cuts that to $3,951. It also left the California income tax out of SALT and skipped the 0.5% charitable floor. It then reported $8,547 on ~$78,017 of taxable income, which single brackets cannot produce (they give about $12,100)." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied qualifying-surviving-spouse joint brackets and the $33,850 joint standard deduction, but that status requires a dependent child, so the head files single. As a single filer, itemized deductions of $29,735.52 beat the $18,150 standard deduction. With the $910 QBI and $3,951 senior deductions, $74,552.59 is taxed at 10%/12%/22% for $11,113.57." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired for 2026 and brought back personal exemptions, the 2%-floor deduction for employee expenses, and 15%/25% brackets, although OBBBA made the TCJA rules permanent. It also made up $13,835 of mortgage interest and used joint surviving-spouse rates without a dependent child. Its submitted $13,013 matches none of its own intermediate results." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse status with a $32,600 joint standard deduction although it requires a dependent child. It left out the senior deduction, the QBI deduction, and the California income tax in SALT. It then reported $6,800, which contradicts its own ~$8,700 estimate; single status with $34,596.57 of deductions gives $11,113.57." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse on joint brackets with the $33,850 joint standard deduction, but without a dependent child the head files single and itemizes $29,735.52. It also took the full $6,000 senior deduction instead of phasing it down by 6% of MAGI over $75,000 to $3,951, and left out the $910 QBI deduction on rental income." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse status with a $32,500 joint standard deduction and joint 10%/12% brackets, but the head has no dependent child and files single. It also skipped the $3,951 phased senior deduction and the $910 QBI deduction, and left California income tax out of SALT." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law for 2026: a $5,300 personal exemption, a 2%-floor deduction for $8,351 of employee expenses, and 15%/25% brackets. OBBBA made the TCJA 10/12/22% schedule and the repeal of those items permanent. It also understated California income tax in SALT at $2,019 (the correct figure is ~$6,059) and left out the $3,951 senior and $910 QBI deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law for 2026: a $5,200 personal exemption, a 2%-floor deduction for employee expenses, and 15%/25% single brackets, although OBBBA made the TCJA rules permanent. It also understated California income tax in SALT and left out the senior and QBI deductions. Its submitted $13,047.69 does not even match the $12,372 in its own explanation." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status but itemized only $24,222 of property tax and charity, leaving out ~$6,059 of California income tax in SALT. It never subtracted the $3,951 phased senior deduction or the $910 QBI deduction, so taxable income was $84,927 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA rules: a $5,000 personal exemption, employee expenses deducted above a 2% floor, and 15%/25% single brackets. OBBBA made the TCJA 10/12/22% schedule permanent and repealed those deductions for good. It also left out the $3,951 phased senior deduction and the California income tax in SALT." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized mortgage interest that is not an input; only the loan balance is listed, so interest is zero. It never accounts for the ~$6,059 California income tax in SALT, the 0.5% charitable floor, the $3,951 senior deduction, or the $910 QBI deduction that together bring taxable income to $74,552.59. Its $11,335 corresponds to about $1,000 less in deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA itemized deductions (~$33,297, including miscellaneous employee expenses), personal exemptions, and pre-TCJA single brackets. OBBBA made the TCJA rules permanent, so 2026 has no personal exemption and no miscellaneous itemized deduction, and uses the 10/12/22% schedule. It also left out the $3,951 senior and $910 QBI deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a $5,050 personal exemption and $6,168 of employee expenses above a 2% floor, and used post-sunset bracket rates; OBBBA permanently eliminated these and kept the 10/12/22% schedule. It also understated California income tax in SALT at $2,172 instead of ~$6,059 and left out the senior and QBI deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. At 2026 single rates, $13,446 corresponds to about $85,150 of taxable income, which is AGI minus only ~$24,000 of property tax and charity. That means it left out the ~$6,059 California income tax in SALT, the $3,951 phased senior deduction, and the $910 QBI deduction, which bring taxable income to $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 limit, but OBBBA raised the 2026 cap to $40,400, so the full $14,887.85 of property tax plus California income tax is deductible. It also left out the $3,951 phased senior deduction for the 77-year-old, so taxable income stayed at $82,846 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $9,260 matches joint-rate (surviving-spouse) tax on about $81,000 of taxable income, after itemizing only property tax and charity. The head has no dependent child and files single, where income above $50,400 is taxed at 22%. Correct single-status deductions of $34,596.57 produce $11,113.57." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $13,200 matches single-rate tax on about $84,000 of taxable income, which is AGI minus only property tax, charity, and QBI. It left out the ~$6,059 California income tax in SALT and the $3,951 phased senior deduction. Its reference to deducting unreimbursed employee expenses is also wrong, because OBBBA permanently eliminated miscellaneous itemized deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2025 single brackets and the 2025 standard deduction instead of 2026 values. It left out the OBBBA senior deduction ($3,951 after phase-down), and it itemized only $24,222, without the ~$6,059 California income tax in SALT or the 0.5% charitable floor. As a result, taxable income was $84,016.75 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $8,351 of unreimbursed employee expenses as an above-the-line deduction, though they are not deductible, and added the $371 tax-exempt pension to AGI. It then subtracted $5,850 of federal withholding that is not an input and is not a nonrefundable credit. Its own arithmetic is also wrong: $84,169 of single taxable income gives about $13,200 of tax, not $7,102." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed no wage income remained taxable after pension and Social Security offsets and that deductions wiped out the tax. In fact, $52,635 of wages, $30,989.41 of pension, and 85% of Social Security are all taxable, giving AGI of $109,149.16. Total deductions are only $34,596.57, leaving $74,552.59 taxable and $11,113.57 of tax." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It offered only a generic description, and its $1,237 implies roughly $12,000 of taxable income. AGI of $109,149.16 minus $34,596.57 of deductions ($29,735.52 itemized, $910 QBI, $3,951.05 senior) leaves $74,552.59 taxable, which is $11,113.57 at 2026 single rates." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets although the head has no dependent child and files single. It also subtracted a personal exemption that OBBBA permanently set to zero. Its ~$30,390 of itemized deductions came from counting employee expenses, which are no longer deductible, rather than the California income tax in SALT." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"At single rates, its $10,247 corresponds to about $70,600 of taxable income, so it took about $3,940 more in deductions than the correct $34,596.57. The senior deduction is only $3,951 after the 6% phase-down above $75,000 MAGI, charity is reduced by the 0.5%-of-AGI floor, and the only other allowable itemized item is $14,887.85 of SALT." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint rates and the joint age-based standard deduction, but that status requires a dependent child. The head files single, itemizes $29,735.52 (more than the $18,150 single senior standard deduction), and owes $11,113.57 at single rates." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It used the surviving-spouse (joint) schedule, but without a dependent child the head files single, where income above $50,400 is taxed at 22%. Its explanation also names no senior or QBI deduction, and it left the California income tax out of the itemized SALT." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, $3,951 senior deduction and $910 QBI deduction all match the correct values. However, it applied surviving-spouse status with joint brackets and the $33,850 joint standard deduction, which requires a dependent child. The head files single, itemizes $29,735.52 instead of taking the $18,150 standard deduction, and owes $11,113.57." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"At single rates, its $9,651 corresponds to about $67,900 of taxable income, so it took about $6,650 more in deductions than the correct $34,596.57. Itemized deductions are limited to $14,887.85 of SALT and charity net of the 0.5% AGI floor, with no mortgage interest and no employee expenses, alongside $910 of QBI and the $3,951 senior deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status and the $3,951 phased senior deduction correctly, but itemized only $26,456, counting just ~$2,234 of California income tax instead of ~$6,059 and skipping the 0.5% charitable floor. It also left out the $910 QBI deduction on rental income, so taxable income was about $78,742 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, $3,951 senior deduction and $910 QBI deduction match the correct values, but it applied surviving-spouse joint brackets and the $33,850 joint standard deduction. That status requires a dependent child, so the head files single, itemizes $29,735.52, and owes $11,113.57 at single rates." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets and a $30,000 standard deduction although that status requires a dependent child. It also left out the $3,951 phased senior deduction, the $910 QBI deduction, and the $29,735.52 of itemized deductions that apply under single status." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law: a $5,419 personal exemption, $6,168 of employee expenses above a 2% floor, and 15%/25% single brackets. OBBBA made the TCJA rules permanent, so none of these apply in 2026. It also understated California income tax in SALT and left out the $3,951 senior and $910 QBI deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status, the phased $3,951 senior deduction and the $910 QBI deduction correctly. However, it put only $1,974 of California income tax into SALT instead of ~$6,059, and it skipped the 0.5%-of-AGI charitable floor. As a result, it itemized $26,196 instead of $29,735.52 and reached taxable income of $78,092 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $1,867 of California income tax in SALT instead of ~$6,059 and set the QBI deduction to $0, although rental income gives a $910 deduction. It also used estimated brackets (10% to $12,225, 12% to $49,675) instead of the 2026 single thresholds of $12,400 and $50,400." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It included 100% of Social Security ($20,735) in AGI instead of the 85% maximum ($17,624.75). It also applied post-sunset 15%/25% brackets and a 2%-floor deduction for employee expenses, though OBBBA made the TCJA rules permanent. On top of that, it understated California income tax in SALT and left out the senior and QBI deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted ~$901 of rental depreciation that is not an input, and it counted only ~$2,024 of California income tax in SALT instead of ~$6,059. It also left out the $910 QBI deduction on rental income, so taxable income was $78,538 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $11,113.57 single-filer liability." -us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status and the $3,951.08 phased senior deduction correctly, but deliberately took no QBI deduction, although 20% of the $4,550 of rental income ($910) applies. It also estimated California income tax at $2,700 instead of ~$6,059 and skipped the 0.5% charitable floor, so taxable income was $78,275.67 instead of $74,552.59." -us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It asserted that the standard deduction left too little taxable income for any tax. In fact AGI is $109,149.16 and total deductions are only $34,596.57, so $74,552.59 is taxable, producing $11,113.57 of tax rather than $0." -us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse using joint brackets, the $33,850 joint standard deduction, and the full $6,000 senior deduction. Without a dependent child the head files single. The senior deduction phases down by 6% of MAGI above $75,000 to $3,951, and the $910 QBI deduction on rental income applies." -us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It computed AGI as $126,773.50 by adding taxable Social Security on top of provisional income, which already includes half of the benefits; correct AGI is $109,149.16. It also made up $12,771 of mortgage interest from the loan balance when no interest is listed, and applied surviving-spouse joint brackets despite having no dependent child." -us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed itemized deductions produced negative taxable income, counting health premiums that fall below the 7.5% floor, nondeductible employee expenses, and made-up mortgage interest. Allowable itemized deductions are only $29,735.52, leaving $74,552.59 taxable and $11,113.57 of tax." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used single status, the phased $3,951 senior deduction and the $910 QBI deduction, and included $8,829 of real estate tax in SALT. The prompt lists no state income tax withheld or paid; the frozen reference adds the engine's $6,058.85 withholding estimate, giving $29,736 of itemized deductions. The model's $24,222 of itemized deductions and $80,066 taxable income reflect a different treatment of that unlisted input and omission of the charitable floor." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse and used the joint brackets and the $32,200 joint standard deduction. That status requires a dependent child, so single status ($18,150 aged standard deduction, single brackets) applies. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 MAGI, and added a non-itemizer charitable deduction, even though itemizing is optimal for a single filer." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used head-of-household status and brackets, which require a qualifying person this filer does not have. It also counted the full $20,735 of Social Security instead of the 85% taxable portion, and subtracted the $8,351 of unreimbursed employee expenses above the line, which is not allowed. On top of that, its reported $11,316 does not match its own $10,564 computation." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated taxable Social Security as about $0 and assumed deductions would wipe out taxable income. With provisional income near $102,000, 85% of benefits ($17,625) is taxable, so AGI is $109,149. After $34,597 of deductions, taxable income is $74,553, not zero." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status with the joint standard deduction and joint brackets, even though the filer has no dependent child, so single status applies. It also left out the phased $3,951 senior deduction. It put no state income tax in SALT, which the prompt does not list; the frozen reference adds the engine's $6,058.85 withholding estimate. As a single filer, the household itemizes under every recorded reading of that input." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It filed with joint brackets as a qualifying surviving spouse even though there is no dependent child. It also made up $12,771 of mortgage interest from the loan balance (no interest paid was listed) and took the full $6,000 senior deduction without the phase-down above $75,000 MAGI. Its final $6,919 does not match its own $7,350 arithmetic." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $24,222 without state income tax in SALT or the charitable floor. The prompt lists no state income tax withheld or paid; the frozen reference's $29,736 itemized total includes the engine's $6,058.85 withholding estimate. It also took the full $6,000 senior deduction instead of phasing it down to $3,951. Its $8,547 does not follow from single brackets applied to its own $78,017 of taxable income, which gives about $11,900." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status with the $32,200 joint standard deduction and joint brackets, but with no dependent child the filer is single. Under single status, itemizing ($29,736 including California income tax in SALT) beats the $18,150 aged standard deduction and the non-itemizer charitable deduction it took, and the 22% bracket applies above $50,400." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and brought back personal exemptions, the 2%-floor deduction for employee expenses, and the 15%/25% brackets. In fact OBBBA made the TCJA structure permanent. It also used QSS joint rates and made up mortgage interest. Its final $13,013 matches none of its own intermediate results ($6,837 to $7,205)." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used joint brackets and the $32,600 joint standard deduction for qualifying-surviving-spouse status, even though the filer has no dependent child. It capped SALT at the old $10,000 instead of the $40,400 OBBBA cap, and left out both the phased senior deduction and the QBI deduction. Its $6,800 does not match its own ~$8,700 computation." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse with joint brackets and the $33,850 joint standard deduction, but single status applies because there is no dependent child. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 MAGI, and it skipped the $910 QBI deduction on the rental income." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It used the surviving-spouse joint standard deduction and joint brackets (2025's $23,850 10% bracket), even though without a dependent child the filer is single. It left out the phased $3,951 senior deduction and the $910 QBI deduction. Its itemized total also left out California income tax in SALT." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law: a personal exemption, the 2%-floor deduction for unreimbursed employee expenses, and 15%/25% brackets. Under OBBBA none of these exist in 2026, and the single brackets are 10%/12%/22%. It used its own $2,019 California tax estimate in SALT; the prompt lists no state income tax paid or withheld, and the frozen reference uses the engine's $6,058.85 withholding estimate. It also left out the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied expired pre-TCJA rules: a $5,200 personal exemption and the 2%-floor deduction for employee expenses, both repealed under OBBBA. It used its own $2,135 California tax estimate in SALT; the prompt lists no state income tax paid or withheld, and the frozen reference uses the engine's $6,058.85 withholding estimate. It also left out the senior and QBI deductions. Its submitted $13,047.69 contradicts its own $12,372 result." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized real estate tax and charity ($24,222), with no state income tax in SALT or charitable floor. The prompt lists no state income tax paid or withheld; the frozen reference adds the engine's $6,058.85 withholding estimate. It also omitted the phased $3,951 senior deduction and $910 QBI deduction, producing $84,927 taxable income compared with $74,553 under the frozen reference's assumptions. Its bracket edges ($12,200/$49,600) are not the 2026 single thresholds." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA law: a $5,000 personal exemption, the 2%-floor deduction for employee expenses, and 10%/15%/25% brackets. In 2026 the single brackets are 10%/12%/22%, and personal exemptions and the 2%-floor deductions are repealed. It also left out the phased senior deduction and the California income tax portion of SALT." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It counted mortgage interest as an itemized deduction even though no interest paid was listed. It also never mentioned the phased $3,951 senior deduction or the $910 QBI deduction. The prompt lists no state income tax paid or withheld; the frozen reference adds the engine's $6,058.85 withholding estimate to SALT. Its $11,335 implies taxable income about $1,000 above the frozen reference's $74,553 under that assumption." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA itemized deductions (including the 2%-floor employee expenses) and personal exemptions, both of which are repealed in 2026 under OBBBA. It also left out the phased senior deduction and the QBI deduction that the reference applies with the 10%/12%/22% single brackets." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired. It deducted $6,168 of unreimbursed employee expenses over a 2% floor and a $5,050 personal exemption, neither of which exists in 2026, and used pre-TCJA rate brackets. It used its own $2,172 California tax estimate in SALT; the prompt lists no state income tax paid or withheld, and the frozen reference uses the engine's $6,058.85 withholding estimate. It also left out the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $13,446 implies taxable income near $85,000 under 2026 single brackets. That is consistent with itemizing real estate tax and charity without state income tax in SALT, the phased $3,951 senior deduction or the $910 QBI deduction. The prompt lists no state tax payment or withholding; $74,553 is the frozen reference's taxable income under the engine's unlisted $6,058.85 withholding estimate." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 rather than the 2026 $40,400 cap. The frozen reference's $14,888 SALT includes the engine's $6,058.85 withholding estimate, although the prompt lists no state tax paid or withheld. It also left out the phased $3,951 senior deduction; its $82,846 taxable income compares with $74,553 under the frozen reference's assumptions." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Under single brackets, its $9,260 implies taxable income near $66,000, about $8,400 below the frozen reference's $74,553. Relative to that reference's assumptions, this is consistent with higher deductions or joint brackets the filer does not qualify for. The reference's $29,736 itemized total includes the engine's $6,058.85 withholding estimate, an input the prompt does not list." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted unreimbursed employee business expenses, which are not deductible in 2026 because OBBBA permanently repealed miscellaneous itemized deductions. It also left out the phased $3,951 senior deduction and the $910 QBI deduction. Its $13,200 implies taxable income near $84,000, compared with the frozen reference's $74,553 under the engine's unlisted $6,058.85 withholding estimate in SALT." +us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $24,222 without state income tax in SALT or the charitable floor; the prompt lists no state income tax paid or withheld, and the frozen reference's $29,736 total adds the engine's $6,058.85 withholding estimate. It also left out the phased $3,951 senior deduction, which applies to filers aged 65+ in 2026. It then used 2025 single brackets ($11,925/$48,475) instead of the 2026 thresholds ($12,400/$50,400)." +us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $8,351 of unreimbursed employee business expenses above the line, which is not allowed, and took the standard deduction even though itemizing ($29,736) is larger. It then subtracted $5,850 of federal withholding that was never given, even though withholding has no effect on tax before refundable credits." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded that deductions fully offset tax and ignored the $109,149 of AGI, which includes $17,625 of taxable Social Security. Under the frozen reference's unlisted withholding assumption, deductions total $34,597, leaving $74,553 taxable income and $11,113.57 tax; the exclusion also records positive tax under the alternative readings." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Its $1,237 implies taxable income of only about $12,000, far below the frozen reference's $74,553, which depends on the engine's unlisted $6,058.85 withholding estimate in SALT. That fits leaving most of the wages, pension and taxable Social Security out of AGI or massively overstating deductions and credits. No nonrefundable credits are available to this filer." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets even though the filer has no dependent child, so single brackets apply. It also subtracted a personal exemption, which is $0 in 2026. With the 22% bracket starting at $50,400, the tax on $74,553 is $11,114, not $9,800." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It used the right structure (itemized deductions, senior deduction, QBI deduction), but its $10,247 implies taxable income near $70,600 under single brackets. That implies about $3,900 more deductions than the frozen reference's $34,597, which includes the engine's unlisted $6,058.85 withholding estimate in SALT. The model gives no deduction amounts that locate the difference." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint rates and the joint standard deduction, but that status requires a dependent child, so the single schedule and the $18,150 aged standard deduction apply. Under single status, itemizing ($29,736) is optimal, and income above $50,400 falls in the 22% bracket." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It taxed itemized-deduction taxable income on the surviving-spouse (joint) schedule, even though the filer has no dependent child, so single brackets apply. It also never mentioned the phased $3,951 senior deduction or the $910 QBI deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It got the phased $3,951 senior deduction and the $910 QBI deduction right, but used the qualifying-surviving-spouse $33,850 joint standard deduction and joint brackets. Without a dependent child, single status applies, with 22% above $50,400. The frozen reference itemizes $29,736 and yields $74,553 taxable income by adding the engine's $6,058.85 withholding estimate to SALT, although the prompt lists no state tax payment or withholding." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $9,651 implies taxable income near $67,900 under single brackets, which implies about $6,600 more deductions than the frozen reference's $34,597 (itemized $29,736 + senior $3,951 + QBI $910). That reference includes the engine's unlisted $6,058.85 withholding estimate in SALT; the model reports a phased senior deduction but gives no itemized or QBI amounts to locate the difference." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated itemized deductions at $26,456. The prompt lists no state tax payment or withholding; the frozen reference instead uses the engine's $6,058.85 withholding estimate for $14,888 of SALT. It also omitted the $910 rental QBI deduction, leaving $78,742 taxable income compared with $74,553 under the frozen reference's assumptions." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It got the phased $3,951 senior deduction and the $910 QBI deduction right, but used surviving-spouse joint brackets and the $33,850 joint standard deduction. With no dependent child the filer is single, with 22% above $50,400. The frozen reference's $29,736 itemized total and $11,113.57 tax depend on the engine's $6,058.85 withholding estimate in SALT, although the prompt lists no state tax paid or withheld." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $30,000 QSS standard deduction and joint brackets, even though the filer has no dependent child and is single. It also left out the phased $3,951 senior deduction and the $910 QBI deduction, and never compared the standard deduction with the larger $29,736 of itemized deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA sunset. It deducted $6,168 of employee expenses above a 2% floor and a $5,419 personal exemption, neither of which is allowed in 2026 under OBBBA. It also left out the phased $3,951 senior deduction and the $910 QBI deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It handled filing status, senior deduction and QBI correctly and used its own $1,974 California tax estimate, giving $10,803 SALT and $78,092 taxable income. The prompt lists no state tax paid or withheld; the frozen reference instead uses the engine's $6,058.85 withholding estimate, giving $14,888 SALT and $74,553 taxable income under that assumption." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It used its own $1,867 California tax estimate for $10,696 SALT; the prompt lists no state tax paid or withheld, and the frozen reference's $14,888 SALT uses the engine's $6,058.85 withholding estimate. It set the QBI deduction to $0, even though 20% of the $4,550 of rental income ($910) is deductible. It also used non-2026 bracket edges ($12,225/$49,675) instead of $12,400/$50,400." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted 100% of Social Security as taxable instead of the 85% cap ($17,625), which inflated AGI to $112,259. It also applied pre-TCJA rules (the 2%-floor deduction for employee expenses and 15%/25% brackets), which do not apply in 2026, and left out the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the 0.5% charitable floor and used its own approximately $2,024 California tax estimate for $10,853 SALT. The prompt lists no state tax paid or withheld; the frozen reference's $14,888 SALT instead uses the engine's $6,058.85 withholding estimate. It also subtracted a rental depreciation that was not given and left out the $910 QBI deduction, which left taxable income at $78,538." +us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no answer and no explanation for federal_income_tax_before_refundable_credits. +us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It used its own $2,700 California tax estimate for $11,529 SALT; the prompt lists no state tax paid or withheld, and the frozen reference's $14,888 SALT uses the engine's $6,058.85 withholding estimate. It also skipped the $910 rental QBI deduction, leaving $78,276 taxable income compared with $74,553 under the frozen reference's assumptions." +us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed taxable income was too low to owe tax and ignored that AGI is $109,149, including $17,625 of taxable Social Security. Even with $34,597 of deductions, taxable income is $74,553, which produces $11,113.57 of tax." +us,scenario_022,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized real estate tax and charity ($24,222), with no state income tax in SALT or charitable floor. The prompt lists no state income tax paid or withheld; the frozen reference adds the engine's $6,058.85 withholding estimate. It also left out the $910 QBI deduction, yielding $80,976 taxable income compared with $74,553 under the frozen reference's assumptions." +us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It filed with joint (QSS) brackets and the joint standard deduction even though there is no dependent child, so single status applies. It took the full $6,000 senior deduction by testing MAGI against the $150,000 joint threshold instead of the single $75,000 threshold, which phases it down to $3,951. It also dropped the $910 QBI deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It used QSS joint brackets (and 2025's $23,850 threshold) even though the filer has no dependent child. It overstated AGI at $126,774 instead of $109,149 through an addition error with Social Security, and made up $12,771 of mortgage interest when no interest paid was listed." +us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed itemized deductions produce negative taxable income. To get there it counted made-up mortgage interest, sub-floor medical costs and nondeductible employee expenses. The frozen reference deducts $34,597 from $109,149 AGI, leaving $74,553 taxable income under the engine's unlisted $6,058.85 withholding estimate in SALT; the exclusion records positive tax under the alternative readings too." us,scenario_022,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare enrollment plus age 77 as sufficient for Medicaid eligibility. Medicare status does not establish Medicaid eligibility, and this person receives no SSI and has medicaid_category NONE, so no California Medicaid pathway applies." us,scenario_022,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used California SDI at 1.2%, which is the 2025 rate, instead of the 2026 rate of 1.3% ($684.26). It then submitted $4,557.61, which does not match even its own $4,658.20 total." -us,scenario_022,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal Social Security and Medicare ($4,026.58). It left out California's mandatory employee SDI contribution of 1.3% of wages ($684.26), which is part of the requested payroll tax." -us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said wrongly that no mandatory California payroll tax applies, so it dropped the 1.3% SDI ($684.26). It also got Medicare wrong ($762.71 instead of $763.21) and then submitted $4,028.27, which does not match its own sum." -us,scenario_022,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Submitted $0 even though its own explanation gives $4,026 of Social Security and Medicare tax. That $4,026 also leaves out the 1.3% California SDI ($684.26), so the right total is $4,710.83." -us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Used 1.2% for California SDI ($631.62), which is the 2025 rate. The 2026 rate is 1.3% ($684.26), so the total came out $52.63 short." -us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Noticed that SDI applies but left it out, wrongly reading the instructions as covering only Social Security and Medicare. The question says to include mandatory employee state payroll taxes, which here is California SDI at 1.3% ($684.26)." -us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Said wrongly that California has no mandatory employee payroll tax, so it left out the 1.3% SDI contribution ($684.26). It also rounded its federal-only total down to $4,026." -us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,Used the 2024 California SDI rate of 1.1% ($578.99). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Said wrongly that California SDI is voluntary or not an employee tax, so it left out the mandatory 1.3% employee contribution ($684.26). It then submitted $3,264.37, which is roughly the Social Security figure alone and does not match its own $4,026.58 FICA total." -us,scenario_022,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Explicitly left out state payroll tax, so it missed California's mandatory 1.3% SDI ($684.26). It also got Medicare wrong ($763.24 instead of $763.21)." -us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,Used the outdated 1.1% California SDI rate ($579). The 2026 rate is 1.3% ($684.26). It also rounded each piece to whole dollars. -us,scenario_022,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Correctly noted that SDI has no wage cap, but used the 2024 rate of 1.1% ($578.99). The 2026 rate is 1.3% ($684.26)." -us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Added only federal Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($684.26). -us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,Used the outdated 1.1% California SDI rate ($579). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,Used the outdated 1.1% California SDI rate ($579). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,Included only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($684.26). -us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,Used the 2023 California SDI rate of 0.9% ($473.72). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,500.29 total implies California SDI at 0.9% (about $473.71), the 2023 rate. The 2026 rate is 1.3% ($684.26)." -us,scenario_022,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Added only federal Social Security and Medicare ($4,026.58) and left out California's mandatory 1.3% employee SDI ($684.26)." -us,scenario_022,payroll_tax,glm-5.2,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Left out California's mandatory 1.3% employee SDI ($684.26). It also rounded its federal-only figure down to $4,026, citing a whole-dollar rounding rule that does not exist." -us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said wrongly that California has no employee payroll tax, so it dropped the 1.3% SDI ($684.26). Its $4,030 is also not the correct federal-only sum of $4,026.58." -us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out California's 1.3% SDI ($684.26). It also got the federal arithmetic wrong: 7.65% of $52,635 is $4,026.58, not $3,308." -us,scenario_022,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,Estimated California SDI at the 2025 rate of 1.2%. The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Applied only the 7.65% combined federal FICA rate and left out California's mandatory 1.3% employee SDI ($684.26). -us,scenario_022,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,Estimated California SDI at the 2025 rate of 1.2%. The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Calculated only federal Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($684.26). -us,scenario_022,payroll_tax,grok-4.5,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($632). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,grok-4.6,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,grok-4.7,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"Used a projected 1.2% California SDI rate ($632), which is the 2025 rate. The 2026 rate is 1.3% ($684.26)." -us,scenario_022,payroll_tax,inkling,llm_error,state_local_rule,False,Used about 1.2% for California SDI ($632). The 2026 employee rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer or explanation for payroll_tax, so there was no value to score." -us,scenario_022,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Correctly applied SDI with no wage cap, but used the 2025 rate of 1.2% ($631.62). The 2026 rate is 1.3% ($684.26)." -us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Included only Social Security and Medicare, rounded to $4,026, and left out California's mandatory 1.3% employee SDI ($684.26)." -us,scenario_022,payroll_tax,ox-alpha,llm_error,state_local_rule,False,Used the 2025 California SDI rate of 1.2% ($632). The 2026 rate is 1.3% ($684.26). -us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Used an outdated 1.1% SDI rate with a wage cap of about $153,164 that no longer exists. In 2026, SDI is 1.3% of all wages ($684.26)." -us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Said wrongly that no mandatory state payroll taxes apply, so it left out California's 1.3% employee SDI ($684.26)." +us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Used the 2025 CA SDI rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). It also submitted $4,557.61, which does not match its own sum of $4,658.20." +us,scenario_022,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal Social Security and Medicare ($4,026.58). It left out California's mandatory 1.3% employee SDI contribution of $684.26 on the $52,635 of wages." +us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said no mandatory California payroll tax applies, so it left out the 1.3% employee SDI ($684.26). It also got Medicare wrong ($762.71 instead of $763.21) and submitted $4,028.27, which does not match its own sum." +us,scenario_022,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Submitted $0 even though its explanation says FICA comes to about $4,026. That FICA-only figure also leaves out California's mandatory 1.3% employee SDI of $684.26." +us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly applied SDI with no wage cap, but used the 2025 rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). That leaves the total $52.64 short." +us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Explicitly excluded CA SDI, even though the question lists mandatory employee state payroll taxes as part of payroll_tax. It therefore left out the 1.3% SDI contribution of $684.26." +us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Said CA has no mandatory employee payroll tax and submitted FICA only ($4,026). It left out the 1.3% CA SDI contribution of $684.26." +us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,Used the 2024 CA SDI rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Called CA SDI voluntary or employer-paid, but it is a mandatory 1.3% employee contribution ($684.26). It also submitted $3,264.37, which is close to Social Security alone and drops the Medicare it had computed." +us,scenario_022,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Included no state payroll tax, so it left out California's mandatory 1.3% employee SDI ($684.26). It also miscomputed Medicare as $763.24." +us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"Used the 2024 SDI rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26), and rounded the FICA components to whole dollars." +us,scenario_022,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,Used the 2025 CA SDI rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,Used the 2025 CA SDI rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Correctly removed the SDI wage cap, but used the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26)." +us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Summed only Social Security and Medicare ($4,027). It left out California's mandatory 1.3% employee SDI contribution of $684.26." +us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,Used the 2024 CA SDI rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,Used the 2024 CA SDI rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Included only federal FICA ($4,027). It left out California's mandatory 1.3% employee SDI of $684.26." +us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,Used the 2023 CA SDI rate of 0.9% ($473.72) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $4,500.29 total implies a CA SDI rate of 0.9% (the 2023 rate, $473.72) instead of the 2026 rate of 1.3% ($684.26)." +us,scenario_022,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Counted only Social Security and Medicare ($4,026.58). It left out California's mandatory 1.3% employee SDI contribution of $684.26." +us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"Correctly treated SDI as mandatory, but used the 2025 rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26)." +us,scenario_022,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Submitted FICA only ($4,026), leaving out California's mandatory 1.3% employee SDI of $684.26. It also rounded the FICA total incorrectly." +us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said CA has no state employee payroll tax, so it left out the mandatory 1.3% SDI contribution of $684.26. Its $4,030 FICA figure is also off from the exact $4,026.58." +us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out California's 1.3% employee SDI ($684.26). Its $3,308 total is also wrong arithmetic, since 7.65% of $52,635 is $4,026.58." +us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,Used the 2025 CA SDI rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Applied only the 7.65% combined FICA rate. It left out California's mandatory 1.3% employee SDI of $684.26. +us,scenario_022,payroll_tax,gpt-6-luna,llm_error,thresholds_rates,False,"Used the 2025 CA SDI rate of 1.2% instead of the 2026 rate of 1.3% ($684.26), and rounded to $4,658." +us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Counted only employee Social Security and Medicare. It left out California's mandatory 1.3% employee SDI of $684.26. +us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,Used the 2025 CA SDI rate of 1.2% ($632) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,Used the 2025 CA SDI rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,Used the 2025 CA SDI rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"Projected a 1.2% CA SDI rate for 2026, but the actual 2026 rate is 1.3% ($684.26). It also assumed an SDI wage base, which no longer exists." +us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,Used about 1.2% for CA SDI ($632) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no payroll_tax value or explanation, so there is no answer to score." +us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"Correctly applied uncapped mandatory SDI, but used the 2025 rate of 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26)." +us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Summed only Social Security and Medicare (about $4,026). It left out California's mandatory 1.3% employee SDI of $684.26." +us,scenario_022,payroll_tax,muse-wonder,llm_error,thresholds_rates,False,Used about 1.2% for CA SDI ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"Used the 2025 CA SDI rate of 1.2% ($632) instead of the 2026 rate of 1.3% ($684.26), and rounded the components to whole dollars." +us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,Used the 2024 CA SDI rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26). It also cited a wage cap that has been repealed. +us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Said no mandatory employee state payroll taxes apply in CA, so it left out the 1.3% SDI contribution of $684.26." us,scenario_022,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It applied joint/surviving-spouse brackets and joint exemption credits, but a head with no dependent files single. It also took the 2% employee-expense floor from CA AGI ($6,521 instead of $6,167.56). Then it dropped its own ~$980 result and reported an unsupported $3,162, when single-schedule tax on $61,134.44 less $306 of credits is $1,968.81." -us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It used the joint schedule and two personal credits plus a senior credit ($459), but with no dependent the head files single, so the credits are $306. It also left out the $6,167.56 deduction for unreimbursed employee business expenses (above the 2%-of-federal-AGI floor), which put taxable income at $67,302 instead of $61,134.44." -us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from a made-up federal taxable income of $134,529 and never removed Social Security to reach the $91,524 CA AGI. It used a standard deduction instead of the $30,390 of CA itemized deductions, then subtracted a fictitious $5,320 'Senior Property Tax Postponement' credit, which is not an income tax credit." -us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed CA taxable income was near zero. In fact CA AGI after removing Social Security is $91,524 and taxable income after $30,390 of itemized deductions is $61,134.44, which produces $2,274.81 of tax less $306 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse and used MFJ brackets on about $66k. With no dependent child the single schedule applies, which gives $2,274.81 on $61,134.44. It also left out the $6,167.56 employee-expense miscellaneous deduction." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used MFJ/surviving-spouse brackets and two personal credits, though the head has no dependent and files single. It also imputed about $7,000 of mortgage interest that was never given. Its own arithmetic ($1,228 less credits) does not support the $1,856 it reported." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out California's miscellaneous itemized deduction for unreimbursed employee business expenses ($8,351 less 2% of federal AGI = $6,167.56), so taxable income came to $67,302 instead of $61,134.44. Its single-bracket tax of $2,393 is also wrong for that income level." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It used joint brackets and two personal credits plus one senior credit ($471), but a surviving spouse with no dependent child files single and gets $306 of credits. It also left out the $6,167.56 employee-expense miscellaneous deduction, so taxable income was $67,302 instead of $61,134.44." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It wrongly concluded that California conforms to the TCJA suspension of unreimbursed employee expenses. California still allows them above a 2%-of-federal-AGI floor ($6,167.56). It also used MFJ brackets and $588 of credits, when the no-dependent surviving spouse files single and gets $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used qualifying-widow(er) brackets, though the head has no dependent and files single. It left out the $6,167.56 employee-expense deduction. It also skipped the $306 personal-plus-senior exemption credits, wrongly saying they were 'embedded in bracket tables'." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It used joint/QSS brackets on $67,300 of taxable income, but the single schedule applies because there is no qualifying dependent. It also left out the $6,167.56 deduction for unreimbursed employee business expenses that California allows above the 2% floor." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $109,149 without subtracting the $17,624.75 of taxable Social Security that California excludes. It also left out the $6,167.56 employee-expense deduction, which inflated taxable income to $84,927 instead of $61,134.44." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It reached the correct $61,134.44 of taxable income. However, it understated single-schedule tax ($2,186 instead of $2,274.81) and subtracted only a $167 personal credit, missing the separate $153 senior exemption credit for a filer aged 65+ (total $306)." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its explanation computes $2,288 less a single $153 credit, which leaves out the $153 senior exemption credit. The submitted $1,964.52 contradicts that explanation and implies bracket tax of $2,270.52 instead of $2,274.81 on $61,134.44, before the $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It counted only charitable gifts and property tax ($24,222) and left out California's miscellaneous itemized deduction for unreimbursed employee business expenses above 2% of federal AGI ($6,167.56). That taxed $67,302 instead of $61,134.44." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It took the 2% employee-expense floor from CA AGI ($91,524) instead of federal AGI ($109,149), overstating the deduction by about $353 ($6,521 instead of $6,167.56). It also took $310 of exemption credits instead of $306, which put the result slightly below the correct figure." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It said it used a single 65+ standard deduction, but itemized deductions of $30,389.97 far exceed the $5,706 CA standard deduction. Its $1,175 does not come from any bracket computation; the correct path is $2,274.81 of tax on $61,134.44 less $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI (itemized $30,743 instead of $30,389.97). Its $2,180 is also higher than single-schedule tax on its own $60,781 of taxable income less the $306 personal and senior credits, so it misapplied the brackets or the credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI instead of federal AGI and taxed $60,781. It then overstated the single-schedule tax ($2,415, when the 2026 brackets give about $2,247 on that income) and took $288 of credits instead of $306." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It showed no derivation and reported $3,211, about $1,242 too high. That gap equals roughly $15k of extra income at the 8% bracket, so it did not tax only the $61,134.44 left after excluding Social Security and taking the $30,390 of itemized deductions, before subtracting the $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI (itemized $30,743 instead of $30,389.97). Its $2,035 also exceeds single-schedule tax on its own $60,781 less $306 of personal and senior credits, so it overstated bracket tax or under-applied the credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation, and its $2,652 is $683 too high, which is about $8.5k of extra income taxed at 8%. That matches leaving out the $6,167.56 employee-expense miscellaneous deduction and under-applying the $306 exemption credits instead of taxing $61,134.44." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It reported $2,050 with no computation, $81 more than single-schedule tax of $2,274.81 on $61,134.44 less $306 of credits. That overstatement equals about $1,000 of extra income at 8%, which matches taking the 2% floor from the wrong AGI or misapplying the brackets." -us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It made up $24,556 of rental income and a mortgage-interest figure, kept taxable Social Security in the base, and used a standard deduction even though itemized deductions of $30,390 are far larger. That produced an $11,626 tax that bears no relation to the $61,134.44 of CA taxable income." -us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added a nonexistent tax-exempt pension adjustment to CA AGI and used a standard deduction with a made-up $1,549 senior add-on. It wrongly said that beat itemized deductions, which are actually $30,390. It also denied the $153 senior exemption credit that applies at age 77." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It said age and deductions wipe out all CA tax. But CA taxable income after excluding Social Security and itemizing $30,390 is still $61,134.44, which produces $2,274.81 of tax, and the exemption credits offset only $306 of it." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave a generic explanation and reported $1,036, about $933 too low. That is consistent with using joint/QSS brackets or extra exemption credits instead of applying single-schedule tax ($2,274.81 on $61,134.44) less $306 of personal and senior credits." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It applied the joint/surviving-spouse schedule ($1,136) and about $486 of joint plus senior credits. With no dependent child the head files single, which gives $2,274.81 of tax less $306 of credits. It also took the 2% employee-expense floor from CA AGI." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It subtracted only the personal exemption credit and missed the $153 senior exemption credit for age 65+. Its $2,893 also overstates single-schedule tax on $61,134.44, which is $2,274.81." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied qualifying-surviving-spouse rates, but that status requires a dependent child and the head has none. The single schedule applies: $2,274.81 on $61,134.44, less $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It computed tax with the surviving-spouse (joint) schedule, but with no qualifying dependent the head files single. It also subtracted only the personal exemption credit, missing the $153 senior credit." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse schedule ($1,159.12) and a joint $306 personal credit plus $153 senior credit. The no-dependent head files single, which gives $2,274.81 less $153 personal and $153 senior credits. It also took the 2% employee-expense floor from CA AGI." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It subtracted only the personal exemption credit and missed the $153 senior exemption credit that a 77-year-old filer receives. That left its result about $191 above the $1,968.81 obtained after the full $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its $2,095 equals single-schedule tax on about $60,781 less only the $153 personal credit. It missed the $153 senior exemption credit and took the 2% employee-expense floor from CA AGI instead of federal AGI." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse brackets and $474 of credits, but a widow(er) without a dependent child files single. Single filing taxes $61,134.44 at $2,274.81 and allows only $306 of personal plus senior credits." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,missing_output,False,"It declined to compute and defaulted to $0. The facts fully determine CA taxable income of $61,134.44, single-schedule tax of $2,274.81, and $306 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It subtracted only the $153 personal exemption credit and missed the $153 senior exemption credit for age 65+. It also took the 2% employee-expense floor from CA AGI, giving $60,781 of taxable income instead of $61,134.44." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got taxable income ($61,134.44) and credits ($306) right but rounded the single-schedule tax to 'about $2,280' instead of computing the exact $2,274.81. That left it $5.19 off." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It took the 2% employee-expense floor from CA AGI ($6,521 instead of $6,167.56), which understated taxable income. Its projected single brackets also understated the tax ($2,173 instead of about $2,247 even on its own $60,781)." -us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,It said no nonrefundable credits apply and so skipped the $153 personal and $153 senior exemption credits ($306). It also took the 2% employee-expense floor from CA AGI instead of federal AGI. -us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It misstated CA AGI as $90,623 instead of $91,524.41 and subtracted only a $154 personal credit, missing the $153 senior exemption credit. Together those errors produced $2,024 instead of $2,274.81 − $306." -us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score." -us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It left out California's miscellaneous itemized deduction for unreimbursed employee business expenses above 2% of federal AGI ($6,167.56), so it taxed $67,302. It also subtracted only a $144 personal credit and missed the senior exemption credit." -us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It concluded income falls below the taxable threshold. In fact CA taxable income after excluding Social Security and itemizing $30,390 is $61,134.44, which carries $2,274.81 of tax before $306 of credits." -us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It wrongly said California taxes federally taxable Social Security and kept CA AGI at $109,149 instead of subtracting $17,624.75. It also left out the $6,167.56 employee-expense deduction and used MFJ brackets for a single filer." -us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $10,726 standard deduction instead of the $30,390 of itemized deductions, wrongly saying California disallows unreimbursed employee expenses. It also used made-up brackets starting at 4% and $639 of credits instead of the single schedule and $306." -us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It said deductions fully offset CA income. But $91,524 of CA AGI less $30,390 of itemized deductions leaves $61,134.44 taxable, which produces $2,274.81 of tax, and the senior and personal credits offset only $306 of it." +us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Treated the head as a qualifying surviving spouse using joint brackets and joint exemption credits, but with no dependent the head files single. It also applied the 2% employee-expense floor to California AGI instead of federal AGI, giving $6,521 instead of $6,168. Its own joint-bracket arithmetic reached about $980, and the final $3,162 is an unsupported jump that does not follow from that computation." +us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Used the joint/surviving-spouse schedule and $459 of credits (two personal plus one senior), but a surviving spouse with no dependent files single: tax is $2,274.81 on the single schedule and the credit is $306. It also left out the $6,168 of employee business expenses above the 2% floor, which California allows, so taxable income came out at $67,302 instead of $61,134." +us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started from an invented $134,529 federal taxable income and applied a standard deduction, never excluding Social Security or itemizing. It then subtracted a made-up $5,320 credit tied to the Senior Property Tax Postponement program, which defers taxes and is not an income tax credit. The correct path is California AGI of $91,524 minus $30,390 itemized, $2,274.81 of single-schedule tax, less the $306 credit." +us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Asserted that California taxable income is near zero. Even with Social Security excluded, California AGI is $91,524, and $30,390 of itemized deductions leaves $61,134 taxable. That produces $2,274.81 of single-schedule tax before the $306 personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"Applied married-filing-jointly brackets and joint exemption credits, but a surviving spouse with no dependent child files single. It also left out the $6,168 of unreimbursed employee expenses above the 2% floor that California still deducts. On the single schedule, $61,134 of taxable income gives $2,274.81, less $306." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Used married-filing-jointly brackets for a surviving spouse with no dependent, who must file single. It added about $7,000 of mortgage interest that was never listed and left out the $6,168 employee-expense deduction. It then reported $1,856, which is above its own $1,228 pre-credit tax, so its credit step added to the tax instead of subtracting." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used single rates and both exemption credits but left out the $6,168 of unreimbursed employee expenses above the 2% federal-AGI floor; California allows this deduction because it did not adopt the TCJA suspension. Itemizing $24,222 instead of $30,390 left taxable income at $67,302 instead of $61,134, and it then also understated the bracket tax on that larger base." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"Applied joint brackets and about $471 of credits (two personal plus one senior), but a surviving spouse with no dependent files single, with a $306 credit. It also dropped the $6,168 employee-expense deduction, using $24,222 of itemized deductions instead of $30,390." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Explicitly assumed California follows the TCJA suspension of unreimbursed employee expenses. California still allows them as miscellaneous itemized deductions above the 2% floor ($6,168). It also used married-filing-jointly brackets and a joint-level personal credit, but a surviving spouse with no dependent files single, giving $2,274.81 less $306." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Used qualifying-widow(er) brackets although the head has no dependent and files single, and left out the $6,168 employee-expense deduction. It guessed a 2.8% effective rate instead of applying brackets. It also subtracted no exemption credits, claiming they were built into the tables, when California grants separate $153 personal and $153 senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"Used joint/qualifying-surviving-spouse brackets, but with no dependent the head files single, and the single schedule on $61,134 gives $2,274.81. It also left out the $6,168 employee-expense deduction (taxable income $67,302 instead of $61,134) and took $285 of credits instead of $306." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $109,149 without subtracting the $17,625 of taxable Social Security that California fully excludes. It also left out the $6,168 employee-expense deduction, so taxable income came out at $84,927 instead of $61,134." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"Reached the correct $61,134 of taxable income but subtracted only a $167 personal exemption credit. It left out the $153 senior exemption credit the 77-year-old head receives, for a total of $306. It also understated single-schedule tax at $2,186 instead of $2,274.81." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used the correct $61,134 of taxable income, but its submitted $1,964.52 corresponds to about $2,270.52 of single-schedule tax less $306. That is $4.29 below the engine's $2,274.81 from the actual bracket thresholds. Its explanation contradicts the submitted value, citing $2,288 of tax and only the $153 personal credit for $2,135." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Left out the $6,168 of unreimbursed employee expenses above the 2% federal-AGI floor, which California allows as a miscellaneous itemized deduction. It itemized $24,222 instead of $30,390, so taxable income came out at $67,302 instead of $61,134, and it took $298 of credits instead of $306." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Applied the 2% floor on unreimbursed employee expenses to California AGI ($91,524) instead of federal AGI ($109,149). That overstated the miscellaneous deduction by $353 and gave taxable income of $60,781 instead of $61,134. It also used $310 of exemption credits instead of $306 ($153 personal plus $153 senior)." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Relied on the standard deduction for a single filer age 65+, but $30,390 of itemized deductions (property tax, charity, and employee expenses above the 2% floor) far exceed the $5,706 standard deduction. Its $1,175 does not come from applying single brackets to the $61,134 of taxable income, which yields $2,274.81 before the $306 credit." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied the 2% employee-expense floor to California AGI instead of federal AGI, itemizing $30,743 instead of $30,390. On its own $60,781 base, single brackets give $2,246.57 and subtracting the $306 credit gives $1,940.57. Its $2,180 is $239 above that, so its bracket-tax and credit steps were misapplied." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied the 2% employee-expense floor to California AGI, getting $60,781 of taxable income instead of $61,134. It then overstated single-bracket tax at $2,415; the correct figure on its own base is $2,246.57. It also took $288 of exemption credits instead of $306." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Gave no derivation. The correct path is California AGI of $91,524 with Social Security excluded, minus $30,390 itemized, for $61,134 taxable, then $2,274.81 on the single schedule less the $306 credit. Its $3,211 overshoots that by $1,242 and does not reflect the single-schedule computation on the correct base." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Applied the 2% employee-expense floor to California AGI instead of federal AGI, itemizing $30,743 instead of $30,390. Against the $2,246.57 single-schedule tax on its own $60,781 base, its $2,035 implies only about $212 of exemption credits instead of the full $153 personal plus $153 senior." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $2,652 matches single-schedule tax on about $67,300 of taxable income less $306. That means it itemized only property tax and charity and left out the $6,168 of unreimbursed employee expenses above the 2% federal-AGI floor, which California allows." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,other,False,"Gave no numeric derivation. The correct path is $61,134 of taxable income (including $6,168 of employee expenses after a 2% federal-AGI floor), $2,274.81 of single-schedule tax, less $306. Its $2,050 corresponds to subtracting only about $225 of exemption credits instead of the full $153 personal plus $153 senior." +us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Invented $24,556 of rental income, a mortgage-interest figure, and $1,000 of taxable Social Security, none of which appear in the inputs. It then used the federal standard deduction instead of California itemized deductions. The correct base is California AGI of $91,524 minus $30,390 itemized, taxed on the single schedule and reduced by the $306 credit." +us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Inflated California AGI to $102,919 by adding a nonexistent $1,750 pension adjustment and not excluding Social Security. It then used a $7,089 standard deduction even though $30,390 of itemized deductions is far larger. It also wrongly said the senior exemption credit does not apply; the 77-year-old head receives $153 personal plus $153 senior." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted that age and deductions eliminate the tax. California AGI is $91,524 even after excluding Social Security, and $30,390 of itemized deductions still leaves $61,134 taxable. That produces $2,274.81 of single-schedule tax, and the $306 of credits cannot offset it." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Gave no specific computation. Its $1,036 is $933 below the result of taxing $61,134 on the single schedule ($2,274.81) and subtracting the $153 personal and $153 senior credits. The figure does not come from the single-filer path that applies to a surviving spouse with no dependent." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Applied the joint/surviving-spouse schedule ($1,136 of tax) and $486 of joint-level credits. A surviving spouse with no dependent files single, so the tax is $2,274.81 on the single schedule less $306. It also applied the employee-expense floor to California AGI, getting $60,781 of taxable income instead of $61,134." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Took only the personal exemption credit and left out the $153 senior exemption credit for a filer age 65 or older. Its $2,893 implies about $68,000 of taxable income, so it also left out the $6,168 of employee expenses above the 2% floor that bring taxable income down to $61,134." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"Applied qualifying-surviving-spouse rates, which roughly double each bracket width, and joint-level exemption credits. The head has no dependent and files single, so tax on $61,134 is $2,274.81 less $306, not the roughly $657 its joint schedule produced." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Used the surviving-spouse (joint) schedule for a head with no dependent child, who must file single. The single schedule applied to $61,134 gives $2,274.81 before the $306 personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Taxed $60,781 on the surviving-spouse schedule ($1,159) and subtracted a joint $306 personal credit plus a $153 senior credit. Without a dependent the head files single, giving $2,274.81 of tax less $153 personal plus $153 senior. It also applied the 2% employee-expense floor to California AGI instead of federal AGI." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,age_disability,False,"Subtracted only the personal exemption credit and left out the $153 senior exemption credit that California grants a 77-year-old filer. Its result is therefore about $153 or more too high compared with $2,274.81 less the $306 total credit." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"Its $2,095 equals single-schedule tax on about $60,800 less only the $153 personal exemption credit, leaving out the $153 senior exemption credit for a filer age 65 or older. It also applied the 2% employee-expense floor to California AGI instead of federal AGI, which overstated the deduction by $353." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse brackets and $474 of joint-level credits for a head with no dependent, who files single with a $306 credit. It also applied the employee-expense floor to California AGI, itemizing $30,742.52 instead of $30,389.97." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Declined to compute and defaulted to zero. The listed facts fully determine the answer: California AGI of $91,524, $30,390 itemized, $61,134 taxable, $2,274.81 of single-schedule tax, less the $306 personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,age_disability,False,"Subtracted only the $153 personal exemption credit and left out the additional $153 senior exemption credit the 77-year-old head receives. It also applied the 2% employee-expense floor to California AGI, giving $60,781 of taxable income instead of $61,134." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Followed the correct path: single filer, $30,390 itemized, $61,134 taxable, and $306 of credits. It estimated single-schedule tax at $2,280 instead of the $2,274.81 the actual bracket thresholds produce, leaving it $5.19 high." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Applied the 2% employee-expense floor to California AGI instead of federal AGI, itemizing $30,743 instead of $30,390. It also used projected 2026 bracket thresholds that produced $2,173 of tax, about $74 below the $2,246.57 the actual single schedule gives on its own $60,781 base." +us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,Stated that no nonrefundable credits apply and reported pre-credit tax. California's $153 personal exemption credit and $153 senior exemption credit together reduce tax by $306. It also applied the 2% employee-expense floor to California AGI instead of federal AGI. +us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,age_disability,False,"Subtracted only the $154 personal exemption credit and left out the $153 senior exemption credit for the 77-year-old head. It also understated California AGI as $90,623 instead of $91,524, which pushed taxable income down to $59,862 instead of $61,134." +us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for state_income_tax_before_refundable_credits, so the required output is missing." +us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Itemized only property tax and charity ($24,222). It left out the $6,168 of unreimbursed employee expenses above the 2% federal-AGI floor, which California allows because it never adopted the TCJA suspension. It then subtracted only a $144 personal credit and dropped the $153 senior exemption credit." +us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Claimed that deductions bring California taxable income to zero. With Social Security excluded, California AGI is $91,524, and $30,390 of itemized deductions leaves $61,134 taxed on the single schedule at $2,274.81 before the $306 credit." +us,scenario_022,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Itemized only $24,222 and left out the $6,168 of unreimbursed employee expenses above the 2% federal-AGI floor that California allows, so it taxed about $67,300 instead of $61,134. It also took $267 of exemption credits instead of $306." +us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Wrongly asserted that California taxes federally taxable Social Security; California excludes all Social Security, so California AGI is $91,524, not $109,149. It also left out the $6,168 employee-expense deduction. It then used married-filing-jointly brackets for a surviving spouse with no dependent, who files single." +us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Used a $10,726 standard deduction instead of itemizing $30,390 (property tax, charity, and employee expenses above the 2% floor, which California allows). It also invented a rate schedule that starts at 4% and reaches 9.3%, when $61,134 of single taxable income tops out in the 8% bracket for $2,274.81 less $306." +us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed that deductions and senior credits reduce the tax to zero. After itemizing $30,390, $61,134 of California taxable income remains, producing $2,274.81 of single-schedule tax. The exemption credits total only $306 ($153 personal plus $153 senior)." us,scenario_022,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly reached about $644 of tax on $6,435 of taxable income, then wiped it out with two credits the household doesn't get. It took a $1,000 Saver's Credit without netting the $8,000 403(b) distribution against contributions under §25B(d)(2), which drops the credit base to $0. It also claimed an AOTC even though qualified tuition is $0." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,other,False,"The model computed $643.50 of tax correctly and correctly set the AOTC to $0. It then applied a $1,000 Saver's Credit without reducing qualified contributions by the $8,000 403(b) distribution in the testing period, which brings eligible contributions and the credit to $0." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left the $8,000 taxable 403(b) distribution out of income entirely and used the outdated $14,600 standard deduction instead of 2026's $16,100. Adding the distribution gives AGI of $22,534 and taxable income of $6,434, or $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model's $0 only works if income is wages alone: $17,443 − $2,778 − $130 = $14,535, which is below the $16,100 standard deduction. It ignored the $8,000 taxable 403(b) distribution, which pushes AGI to $22,534 and taxable income to $6,434, or $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model computed the ~$644 tentative tax correctly, then offset it with a $1,500 nonrefundable AOTC. The AOTC is 100%/25% of qualified tuition and fees, and none are listed, so the credit is $0 and the $643 tax stands." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed the head had qualified 1098-T expenses and applied the nonrefundable AOTC. Enrollment and a 1098-T don't create the credit. With $0 qualified tuition the AOTC is $0, so the $643 tax is not reduced." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model applied up to $1,500 of nonrefundable AOTC against the ~$644 tax. No qualified tuition or fees are listed, so the AOTC base and credit are $0 and the tax stays at $643." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"The model explicitly assumed the $8,000 403(b) distribution doesn't reduce eligible contributions, which gave a $1,000 Saver's Credit. Under §25B(d)(2), distributions received in the testing period reduce qualified contributions, and $8,000 exceeds the $3,599 contributed, so the credit is $0 and $643 of tax remains." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model got AGI and the zero nonrefundable credits right but estimated the 2026 single standard deduction as $15,350 by inflating the 2025 figure. The OBBBA-set 2026 amount is $16,100. That leaves $6,434 of taxable income and $643 of tax, not $719." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misadded gross income and claimed taxable income was near $0 after the standard deduction, when it is $6,434. It then used a nonrefundable AOTC to erase the rest, even though $0 qualified tuition makes the AOTC $0." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"The model computed ~$644 of tax correctly but applied a 50% Saver's Credit on $2,000 of contributions. It never netted the $8,000 403(b) distribution against contributions as §25B(d)(2) requires, which makes the credit $0." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer-sponsored insurance premiums from wages as if they were employee pre-tax payroll deductions, cutting AGI to $14,446. Only the 401(k) deferral reduces wages here, so AGI is $22,534. The model also used the stale $15,750 standard deduction instead of $16,100." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the $8,089 employer insurance premiums from wages, giving AGI of $14,446 instead of $22,534. It also used an $8,350 pre-TCJA standard deduction instead of the OBBBA 2026 amount of $16,100." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and used an $8,300 standard deduction plus a $5,300 personal exemption. OBBBA made TCJA permanent, so 2026 has a $16,100 single standard deduction and no personal exemption, which gives $6,434 of taxable income and $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"The model used the 2025 $15,750 standard deduction instead of $16,100. It then applied a $1,000 Saver's Credit without reducing contributions by the $8,000 403(b) distribution in the testing period, which zeroes out that credit." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"The model used a pre-OBBBA standard deduction plus personal exemption of about $14,000 instead of the $16,100 standard deduction with no exemption. It then claimed a 50% Saver's Credit while ignoring that the $8,000 403(b) distribution wipes out qualified contributions under §25B(d)(2)." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model said taxable income is $0 after deductions. AGI is $22,534, including the $8,000 taxable 403(b) distribution, which exceeds the $16,100 standard deduction by $6,434 and produces $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model used the Saver's Credit to offset all the tax but didn't reduce qualified contributions by the $8,000 403(b) distribution. That distribution exceeds total contributions, so the credit is $0 and $643 of tax remains." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer insurance premiums from wages, giving AGI of $14,446 instead of $22,534. It then applied a Saver's Credit that the $8,000 403(b) distribution eliminates under the §25B(d)(2) distribution offset." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model's $0 relies on nonrefundable credits wiping out the tax. With $6,434 of taxable income ($643 of tax), the only candidates are the Saver's Credit, which the $8,000 403(b) distribution reduces to $0, and the AOTC, which is $0 with no qualified tuition. Nothing offsets the $643." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model's $753.50 tentative tax implies the stale $15,000 standard deduction instead of $16,100. It then subtracted a $200 Saver's Credit, which should be $0 because the $8,000 403(b) distribution exceeds total contributions under §25B(d)(2)." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model said taxable income is $0. Wages net of the 401(k) deferral plus the $8,000 taxable 403(b) distribution, less the $130 IRA deduction, give AGI of $22,534, which exceeds the $16,100 standard deduction by $6,434 and produces $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model got AGI and the zero Saver's Credit and AOTC right, but its $754 implies about $7,540 of taxable income, meaning it used the pre-OBBBA $15,000 standard deduction. The 2026 amount is $16,100, which gives $643." -us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model handled AGI and both credits correctly but estimated the 2026 single standard deduction at $15,400 instead of the OBBBA amount of $16,100. That overstated taxable income by $700 and the tax by $70." -us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model added back the 401(k) deferral and invented a $500 exclusion on the distribution, then used a $15,000 standard deduction. Its reasoning concluded $0 but it submitted an impossible negative $-997, when federal income tax before refundable credits can't go below zero. The correct path gives AGI of $22,534, taxable income of $6,434, and $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used an AOTC nonrefundable credit to eliminate the remaining tax. The AOTC requires qualified tuition and fees, none are listed, so the credit is $0 and $643 of tax on $6,434 of taxable income remains." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model applied a partial nonrefundable AOTC based only on enrollment status and appears to have left out the $8,000 403(b) distribution. No qualified tuition is listed, so the AOTC is $0, and the correct $22,534 AGI and $16,100 standard deduction give $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The model used the Saver's Credit to offset the tax without netting the $8,000 403(b) distribution against the listed contributions. Under §25B(d)(2) that distribution reduces qualified contributions to $0, so no credit applies and $643 of tax remains." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,other,False,"The model applied a nonrefundable Saver's Credit that ignores the testing-period distribution reduction. The $8,000 403(b) distribution exceeds the $3,599 of contributions, so the credit is $0 and the $643 tax is unreduced." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer-sponsored insurance premiums as pretax wage reductions, giving AGI of $14,446. Only the $2,778 401(k) deferral reduces wages, so AGI is $22,534 and $6,434 remains taxable after the $16,100 standard deduction." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated income as wages below the standard deduction and ignored the $8,000 taxable 403(b) distribution. Including it gives AGI of $22,534, taxable income of $6,434, and $643 of tax, with no nonrefundable credits available." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed TCJA expired and used an ~$8,500 standard deduction plus a ~$5,300 personal exemption. OBBBA made TCJA permanent, so 2026 has a $16,100 standard deduction and no exemption, giving $643 of tax rather than $860." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"The model assumed a standard deduction plus a personal exemption, though OBBBA made the exemption $0 and set the 2026 standard deduction at $16,100. It then applied a $1,000 Saver's Credit while ignoring that the $8,000 403(b) distribution reduces qualified contributions to $0." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model correctly zeroed out the Saver's Credit and AOTC but assumed TCJA expired, using a $8,500 standard deduction and a $5,450 personal exemption. Under OBBBA the 2026 single standard deduction is $16,100 with no exemption, which gives $6,434 of taxable income and $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA expired, using an $8,500 standard deduction and pre-TCJA 10%/15% brackets. That gave $14,035 of taxable income and $1,485 of tax, instead of the $16,100 OBBBA standard deduction that gives $6,434 taxed entirely at 10%." -us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer health premiums from wages, giving AGI of $14,446, which falls below the $16,100 standard deduction. The premiums are not employee pretax deductions here, so AGI is $22,534 and $643 of tax remains." -us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for federal_income_tax_before_refundable_credits. The correct derivation gives $6,434 of taxable income taxed at 10%, or $643, with no nonrefundable credits." -us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value for federal_income_tax_before_refundable_credits. The correct derivation gives $6,434 of taxable income taxed at 10%, or $643, with no nonrefundable credits." -us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model used a $15,400 standard deduction instead of $16,100. It then applied a $1,500 nonrefundable AOTC even though no qualified tuition is listed, which makes the AOTC $0 and leaves $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model correctly reached ~$644 of tax and a $0 AOTC but computed the Saver's Credit on $2,908 of contributions without subtracting the $8,000 403(b) distribution received in the testing period. That subtraction drives eligible contributions and the credit to $0." -us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model used the stale $15,750 standard deduction and then offset the tax with a nonrefundable AOTC. No qualified tuition or fees are listed, so the AOTC is $0, and the $16,100 standard deduction leaves $643 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model said income is too low to owe tax after the standard deduction and leaned on the AOTC. AGI of $22,534 exceeds the $16,100 standard deduction by $6,434, giving $643 of tax, and the AOTC is $0 with no qualified tuition." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly got $644 of tentative tax, then erased it with a $1,000 Saver's Credit and an AOTC. The $8,000 of 403(b) distributions reduces the $3,599 of eligible contributions to $0 on Form 8880. The AOTC is $0 because no qualified tuition is listed. Tax should stay at $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,other,False,"The model computed the $643.50 tentative tax correctly but applied a $1,000 Saver's Credit. It ignored the §25B(d)(2) rule that testing-period distributions ($8,000 from the 403(b)) are subtracted from qualified contributions, which leaves a zero credit base and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left the $8,000 of taxable 403(b) distributions out of income and used the outdated $14,600 standard deduction, which produced negative taxable income. Correctly, AGI is $22,534, the standard deduction is $16,100, and taxable income is $6,434, giving $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model only subtracted the 401(k)/IRA deferrals and the standard deduction from wages. Its zero result is consistent with dropping the $8,000 taxable 403(b) distribution, which pushes AGI to $22,534 and leaves $6,434 taxable after the $16,100 standard deduction, or $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly got about $644 of tax but offset it with a $1,500 nonrefundable AOTC. With no qualified tuition or fees listed, qualified education expenses are $0 and so is the AOTC, so tax should stay at $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model assumed 'qualified 1098-T expenses' that were never listed and claimed the nonrefundable AOTC against the $644 tax. Unlisted tuition is $0, so the AOTC is $0, no credit applies, and the result is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reached about $644 of tax, then applied up to $1,500 of nonrefundable AOTC. The AOTC requires qualified education expenses, and none are listed, so the credit is $0 and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"The model explicitly assumed that the 403(b) distributions do not reduce eligible Saver's Credit contributions. That is wrong: Form 8880 line 4 subtracts the $8,000 of testing-period distributions from the $3,599 of contributions, which zeroes the credit and leaves $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly found AGI of $22,535 and no nonrefundable credits, but it used an inflation guess of $15,350 for the 2026 single standard deduction. The OBBBA amount is $16,100, which gives $6,434 of taxable income and $643.43 of tax, not $719." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model claimed taxable income is near zero and that the AOTC wipes out any remainder. In fact, $22,534 of AGI less the $16,100 deduction leaves $6,434 taxable. The AOTC is $0 because no qualified expenses are listed, so tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"The model correctly computed about $644 of tax but applied a $1,000 Saver's Credit at the 50% rate. It missed that the $8,000 403(b) distribution in the testing period reduces qualified contributions to $0 under §25B(d)(2)." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer-sponsored insurance premium from wages as a pre-tax exclusion, but the listed gross wages are not reduced by that premium. Without that subtraction, AGI is $22,534, taxable income is $6,434, and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the $8,089 ESI premium from wages, which understated AGI at $14,446. It also used a TCJA-expiration standard deduction of $8,350 instead of the $16,100 2026 OBBBA amount. The correct AGI of $22,534 less $16,100 gives $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed TCJA had expired and used an $8,300 standard deduction plus a $5,300 personal exemption. OBBBA made the higher deduction permanent ($16,100 single for 2026) and kept the exemption at $0, so taxable income is $6,434 and tax is $643.43, not $893.50." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"The model used the 2025 $15,750 standard deduction instead of $16,100. It then applied a $1,000 Saver's Credit without netting the $8,000 403(b) distribution against contributions, which zeroes the credit and leaves $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"The model used a TCJA-expiration deduction-plus-exemption of about $14,000 instead of the $16,100 2026 standard deduction. It then applied a $1,000 Saver's Credit, ignoring that the $8,000 of 403(b) distributions eliminates the credit's contribution base. No credit applies, and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model asserted zero taxable income, but AGI of $22,534 (including the $8,000 taxable 403(b) distribution) exceeds the $16,100 standard deduction by $6,434. That remainder is taxed at 10% for $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model offset the remaining tax with the Saver's Credit. The $8,000 of 403(b) distributions in the testing period exceeds the $3,599 of retirement contributions, so the credit is $0 and the $643.43 tax on $6,434 of taxable income stands." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reached its $14,446 AGI by excluding the $8,089 ESI premium from wages, which is not a pre-tax reduction in this input. It then applied a Saver's Credit that the $8,000 403(b) distribution eliminates. The correct AGI of $22,534 yields $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model claimed nonrefundable credits reduce tax to $0. No nonrefundable credit applies: the AOTC has no qualified expenses, and the Saver's Credit base is zeroed by the $8,000 403(b) distribution. That leaves 10% of $6,434 of taxable income, or $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its $753.50 pre-credit tax implies the 2025 $15,000 standard deduction instead of $16,100. It then subtracted a $200 Saver's Credit even though the $8,000 403(b) distribution reduces eligible contributions to $0. The correct result is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model claimed zero taxable income, which is consistent with omitting the $8,000 taxable 403(b) distribution. Including it gives AGI of $22,534 and $6,434 of taxable income after the $16,100 standard deduction, or $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly zeroed the AOTC and Saver's Credit, but its $754 tax implies the 2025 $15,000 standard deduction. The 2026 single amount is $16,100, which gives $6,434 of taxable income and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model handled AGI and credits correctly but used an estimated $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100. That overstated taxable income at $7,135 instead of $6,434, giving $713.50 instead of $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model refused to exclude the traditional 401(k) deferral from wages, invented a $500 distribution exclusion, used a $15,000 standard deduction, and gave a negative value. Tax after nonrefundable credits cannot go below zero. The correct derivation gives $6,434 taxable and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model used an AOTC to eliminate the remaining tax, but no qualified education expenses are listed, so the AOTC is $0. The $6,434 of taxable income left after the $16,100 standard deduction produces $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model subtracted a partial nonrefundable AOTC from the tax. The AOTC is $0 with no qualified tuition listed, so no credit reduces the $643.43 tax on $6,434 of taxable income." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The model applied the Saver's Credit to offset the tax. It missed the Form 8880 line 4 reduction: $8,000 of 403(b) distributions exceeds the $3,599 of contributions, so the credit is $0 and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,other,False,"The model offset the tax with the Saver's Credit, but the $8,000 taxable 403(b) distribution in the testing period reduces qualified contributions to $0 under §25B(d)(2). No credit applies and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model reached its $14,446 AGI by treating the $8,089 employer-sponsored insurance premium as a pre-tax wage exclusion, which the input does not support. With AGI of $22,534, taxable income is $6,434 after the $16,100 deduction, and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated income as being under the standard deduction and ignored the $8,000 taxable 403(b) distribution. With it, AGI is $22,534, which exceeds the $16,100 deduction by $6,434. That amount is taxed at 10% for $643.43, with no offsetting nonrefundable credits." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed TCJA expiration, with an $8,500 standard deduction and a $5,300 personal exemption. OBBBA set the 2026 single standard deduction at $16,100 with no personal exemption, so taxable income is $6,434 and tax is $643.43, not $860." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"The model used TCJA-expiration rules (a lower standard deduction plus a personal exemption) instead of the $16,100 2026 deduction. It then applied a $1,000 Saver's Credit that the $8,000 403(b) distribution eliminates. No credit applies, and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model correctly zeroed the Saver's Credit and AOTC, but it assumed TCJA expired and used an $8,500 deduction plus a $5,450 exemption. The 2026 law is a $16,100 standard deduction and a $0 exemption, which gives $6,434 taxable and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied pre-TCJA 2026 rules: an $8,500 standard deduction and a 15% bracket above about $12,495. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent, so $6,434 of taxable income is taxed at 10% for $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 employer health premium from wages to reach $14,446 of AGI, but the listed gross wages are not reduced by that premium. Correct AGI is $22,534, leaving $6,434 taxable after the $16,100 deduction and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for this output. The correct derivation is AGI of $22,534 less the $16,100 standard deduction, giving $6,434 taxed at 10% for $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for this output. The correct derivation is AGI of $22,534 less the $16,100 standard deduction, giving $6,434 taxed at 10% for $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model used a $15,400 standard deduction instead of $16,100, then offset the tax with a $1,500 AOTC. No qualified tuition is listed, so the AOTC is $0, and the $6,434 of taxable income produces $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model correctly applied no credits, but its $6,470 taxable income implies a deduction of about $16,065 rather than the exact $16,100 2026 single standard deduction. Taxable income should be $6,434.34, which gives $643.43 instead of $647." +us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model computed $644 of tax correctly but applied a $1,000 Saver's Credit on $2,908 of contributions. It never subtracted the $8,000 of testing-period 403(b) distributions, which reduces the credit base to $0 and leaves $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used the 2025 $15,750 standard deduction instead of $16,100. It then offset the tax with an AOTC even though no qualified education expenses are listed, so the AOTC is $0 and tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model claimed income is too low for tax after the standard deduction and relied on the AOTC. In fact, $22,534 of AGI exceeds the $16,100 deduction by $6,434, and the AOTC is $0 with no qualified expenses, so tax is $643.43." us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly phased the EITC out to $0 and even noted that zero listed tuition gives a $0 AOTC. It then reversed itself by assuming 'typical qualified expenses default' of at least $4,000 and submitted a $1,000 refundable AOTC. With no qualified expenses, the AOTC is $0." us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the full $2,500 AOTC as refundable. The refundable share is capped at 40% ($1,000). More importantly, the credit base is $0 because no qualified tuition expenses are listed. It also left the $8,000 403(b) distribution out of AGI." us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model made up a refundable AOTC of about $400 plus a small EITC to reach $522. No qualified education expenses are listed, so the AOTC is $0. Its own AGI of $22,535 is above the roughly $19,540 point where the 2026 childless EITC reaches zero, so the EITC is also $0." @@ -1736,31 +1762,32 @@ us,scenario_023,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical us,scenario_023,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, violating the required submission contract." us,scenario_023,head_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated California CHIP as covering disabled adults up to an extended 266% FPL threshold. It failed to apply the categorical exclusion for a person already Medicaid-eligible through the WORKING_DISABLED_BUY_IN pathway, which makes this head ineligible for CHIP regardless of the CHIP income thresholds." us,scenario_023,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable output for head_chip_eligible. The required result was 0 because Medicaid eligibility under WORKING_DISABLED_BUY_IN categorically excludes the head from CHIP. -us,scenario_023,head_medicaid_eligible,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"The model saw MAGI of about $22,535 above 138% FPL and found no SSI receipt, then said not eligible. It never applied the working-disabled buy-in (California's 250% Working Disabled Program), which covers this employed, disabled head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"The model submitted 0, which contradicts its own explanation saying eligible with value = 1. That explanation also rested on the wrong pathway: it claimed income was well under 138% FPL, but MAGI is 1.41x FPL, so the adult expansion fails. Eligibility comes only from the working-disabled buy-in, which the model never mentioned." -us,scenario_023,head_medicaid_eligible,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"The model treated the 138% FPL Medi-Cal adult expansion as the only test and ruled the head out because income exceeded it. It ignored that the head is disabled and working, which qualifies them for the working-disabled buy-in at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"The model said no other Medi-Cal pathway applied after MAGI exceeded 138% FPL. That is wrong: the head is disabled and employed, and the working-disabled buy-in (250% FPL limit) covers them at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"The model applied only the 138% FPL single-adult expansion limit. It never considered the working-disabled buy-in category, which covers the disabled, working head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"The model checked only the 138% FPL expansion limit and SSI-linked Medicaid. It missed the working-disabled buy-in, which needs no SSI receipt and covers the disabled, employed head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"The model said not eligible because MAGI exceeded the 138% FPL expansion threshold, and never used the head's disability and employment. Those facts put the head in the working-disabled buy-in category, under whose limits 1.41x FPL qualifies." -us,scenario_023,head_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"The model looked only at the 138% FPL expansion test and SSI-based non-MAGI pathways. It left out the working-disabled buy-in, which covers this employed, disabled head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"The model correctly found MAGI ($22,535) just above 138% FPL and countable income above the SSI/SSP standard. But it treated SSI-linked coverage as the only disability pathway and never applied California's Working Disabled Program (250% FPL limit, with a $130 asset balance far below any resource limit), under which the head qualifies." -us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"The model applied only the adult income limit and called the head not eligible. It never evaluated the working-disabled buy-in category, which covers this disabled, working head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"The model brushed aside the stated 'is disabled' fact and gave no income-based reason for ineligibility. A disabled head with wages falls into the working-disabled buy-in category, and 1.41x FPL is within its limits." -us,scenario_023,head_medicaid_eligible,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"The model compared income only with the adult Medi-Cal (138% FPL) limit. It never applied the working-disabled buy-in, whose higher income ceiling covers the disabled, employed head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"The model said no separate qualifying category was established, but the listed facts that the head is disabled and earns wages do establish the working-disabled buy-in category. That category covers the head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"The model concluded that disability without SSI gives no Medicaid pathway. In fact, disability plus employment qualifies the head for the working-disabled buy-in (250% FPL limit), which covers them at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"The model tested only the California adult expansion income limit. It skipped the working-disabled buy-in, which covers the disabled, employed head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"The model said no separate pathway was specified, overlooking that the head's disability and wages trigger the working-disabled buy-in. Under that category the head is eligible at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"The model checked only the adult MAGI limit and SSI receipt. The working-disabled buy-in requires neither SSI receipt nor income at or below 138% FPL, and it covers this disabled worker at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"The model rejected both the 138% FPL expansion and the SSI-related aged/blind/disabled pathway on income. It never applied the working-disabled buy-in, whose 250% FPL ceiling covers the employed, disabled head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"The model limited its check to MAGI expansion and SSI receipt. It never considered the working-disabled buy-in, which covers the disabled, working head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"The model ruled out the expansion and the SSI-level aged/blind/disabled pathway on income. It missed the working-disabled buy-in, whose income limit is far above SSI levels and covers the head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,No value or explanation was returned for head_medicaid_eligible. The correct answer is eligible through the working-disabled buy-in category. -us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,No value or explanation was returned for head_medicaid_eligible. The correct answer is eligible through the working-disabled buy-in category. -us,scenario_023,head_medicaid_eligible,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"The model applied only the 138% FPL adult limit and the SSI-recipient pathway. It never considered the working-disabled buy-in, which covers the disabled, employed head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"The model overstated MAGI at $25,443 by not subtracting the $2,778 traditional 401(k) deferral. It also rejected the aged/blind/disabled pathway on income without applying the working-disabled buy-in, whose 250% FPL ceiling covers the head at 1.41x FPL." -us,scenario_023,head_medicaid_eligible,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"The model compared income with the general Medi-Cal adult limit only. It ignored the working-disabled buy-in category, which covers the disabled, working head at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"The model correctly found that MAGI of $22,535 is above 138% FPL, but it only checked expansion and SSI receipt. It never applied the working-disabled buy-in, which covers this employed, disabled head at up to 250% FPL." +us,scenario_023,head_medicaid_eligible,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"The explanation says the head is eligible (value = 1), but the submitted value was 0, so the explanation and the answer contradict each other. The explanation's reasoning is also wrong: it puts income well under 138% FPL when MAGI is 1.41x FPL. Eligibility actually comes from the working-disabled buy-in category, not expansion." +us,scenario_023,head_medicaid_eligible,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"The model treated the 138% FPL adult-expansion limit as the only Medi-Cal test. It ignored the head's disability and employment, which qualify them for the working-disabled buy-in, a category that allows income well above 138% FPL." +us,scenario_023,head_medicaid_eligible,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"The model said no other Medi-Cal pathway qualifies. That missed the working-disabled buy-in, which the employed, disabled head meets at 1.41x FPL because its income limit is 250% FPL, not 138%." +us,scenario_023,head_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,The model only compared MAGI to the 138% FPL expansion limit. It never considered the working-disabled buy-in category that covers a disabled worker at 1.41x FPL. +us,scenario_023,head_medicaid_eligible,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"The model ruled out expansion on MAGI and the SSI-linked pathway on no SSI payment, then stopped. It skipped the working-disabled buy-in, which gives Medicaid to employed disabled adults without SSI receipt at incomes above 138% FPL." +us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"The model used only the 138% FPL expansion threshold. It ignored that the disabled, employed head qualifies under the working-disabled buy-in, whose income limit is well above 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"The model checked expansion and the SSI-based non-MAGI pathway but left out the working-disabled buy-in. That category makes this employed, disabled head eligible at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"The model's MAGI of $22,535 is correct, but it only tested 138% FPL expansion and SSI/SSP-standard ABD eligibility. It missed the working-disabled buy-in, which covers disabled workers at up to 250% FPL and places this head in Medicaid." +us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,The model applied only the adult-income Medicaid test and gave no weight to the head's disability and employment. Those two facts qualify the head for the working-disabled buy-in. +us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"The model mentioned the disability but brushed it aside. It never applied the working-disabled buy-in, which makes an employed, disabled adult at 1.41x FPL eligible for Medicaid." +us,scenario_023,head_medicaid_eligible,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"The model judged eligibility only against the adult Medi-Cal (138% FPL) income limit. It never considered the working-disabled buy-in category, which the employed, disabled head satisfies." +us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"The model said no separate qualifying category was established by the facts. In fact, the listed disability plus wage employment establish the working-disabled buy-in category, which grants eligibility at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"The model correctly said disability alone does not create SSI-linked eligibility. It still missed that disability combined with employment qualifies the head for the working-disabled buy-in, which has a 250% FPL income limit." +us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"The model tested only the adult-expansion income limit. It overlooked the working-disabled buy-in pathway, which covers this disabled worker at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"The model said no other coverage pathway was specified. However, the head's disability and employment trigger the working-disabled buy-in, which allows income above the 138% FPL expansion limit." +us,scenario_023,head_medicaid_eligible,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"The model required either adult-expansion income or SSI receipt. It missed the working-disabled buy-in, which needs neither and covers the employed, disabled head at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"The model ruled out the ABD pathway using SSI-related income limits. It never applied the working-disabled buy-in, a separate disabled-worker category with a 250% FPL limit that this head meets." +us,scenario_023,head_medicaid_eligible,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"The model checked only 138% FPL expansion and SSI receipt. It missed the working-disabled buy-in category, which makes the employed, disabled head eligible." +us,scenario_023,head_medicaid_eligible,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,The model rejected the ABD pathway on SSI-level income limits but never tested the working-disabled buy-in. That category's 250% FPL limit easily covers income at 1.41x FPL. +us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model gave no answer for head_medicaid_eligible. The correct result is eligible through the working-disabled buy-in category. +us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model gave no answer for head_medicaid_eligible. The correct result is eligible through the working-disabled buy-in category. +us,scenario_023,head_medicaid_eligible,minimax-m3,prompt_ambiguity,categorical_eligibility,False,The model applied only the 138% FPL expansion limit and SSI receipt. It never considered the working-disabled buy-in that covers this disabled worker at 1.41x FPL. +us,scenario_023,head_medicaid_eligible,muse-wonder,prompt_ambiguity,categorical_eligibility,False,"The model required either adult-expansion income or an SSI-linked disability payment. It missed the working-disabled buy-in, which gives employed disabled adults Medicaid without SSI and above 138% FPL." +us,scenario_023,head_medicaid_eligible,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"The model left the traditional 401(k) deferral out of MAGI, putting it at $25,443 instead of $22,535. It also treated the head as non-disabled for expansion purposes and missed the working-disabled buy-in, which makes the employed, disabled head eligible." +us,scenario_023,head_medicaid_eligible,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"The model applied only the standard adult income limit. It ignored the working-disabled buy-in category, which the employed, disabled head qualifies for at 1.41x FPL." us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the `is_disabled` flag as a stand-in for SSDI-based Medicare eligibility. Medicare for someone under 65 requires 24 months of SSDI entitlement or ESRD, and this head has $0 in Social Security disability benefits and no ESRD. At age 28 with no SSDI history, the head is ineligible, so the correct value is 0." us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model said disability alone makes someone under 65 eligible for Medicare. It skipped the requirement of 24 months of SSDI entitlement or an ESRD diagnosis, and the head meets neither because they have no SSDI income. The head is 28 and not eligible, so the correct value is 0." us,scenario_023,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied a disability-based Medicare rule using only the `is_disabled` flag. It ignored that the disability pathway requires 24 months of SSDI entitlement or ESRD, and the head has no Social Security disability income. At age 28 without SSDI or ESRD, the head is ineligible, so the correct value is 0." @@ -1768,193 +1795,197 @@ us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_ou us,scenario_023,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used 1.13% as the CA SDI rate ($197.11) instead of the 2026 rate of 1.3%, which gives $226.75. That understated the state payroll tax by about $30. The FICA amounts were correct." -us,scenario_023,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal FICA ($1,334.39) and left out California's mandatory employee SDI. At 1.3% with no cap, SDI adds $226.75 on $17,443 of wages." -us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and so left out CA SDI. SDI is a mandatory 1.3% withholding on all wages ($226.75). The model also replaced its own $1,334.38 FICA sum with an unexplained $1,335.85." -us,scenario_023,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Applied only the 7.65% FICA rate and left out California's mandatory employee SDI. At 1.3% of $17,443, SDI adds $226.75." -us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,Used the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75). This understated the total by about $17. -us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Used 0.9% for CA SDI ($156.99), an outdated rate, instead of the 2026 rate of 1.3% ($226.75). It then submitted $1,467.32, which does not match its own $1,490.38 sum." -us,scenario_023,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Used the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75). It also rounded its $1,543.71 sum to $1,546 for no stated reason." -us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Assumed the CA SDI rate was 1.1% for 2026 ($191.87). The 2026 rate is 1.3% ($226.75), so SDI was understated by about $35." -us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Wrongly treated California as having no mandatory employee state payroll tax and left out CA SDI. SDI is withheld at 1.3% on all wages, which is $226.75 here." -us,scenario_023,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Explicitly excluded CA SDI, even though the question asks for mandatory employee state payroll taxes. This dropped the $226.75 SDI contribution (1.3% of wages)." -us,scenario_023,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Subtracted the $8,089 employer-sponsored insurance premium from FICA wages, even though the stated $17,443 of gross wages is the payroll-tax base. It also left out CA SDI (1.3%, $226.75) completely." -us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Reduced the payroll-tax base to $9,354 by subtracting the $8,089 ESI premium, although all taxes apply to the full $17,443 of wages. It also used a 1.2% CA SDI rate instead of the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Used a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75). The FICA amounts were correct. -us,scenario_023,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied a combined 8.85% rate, which uses the 2025 CA SDI rate of 1.2%. The correct combined rate is 8.95%, with 2026 SDI at 1.3%." -us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,Estimated CA SDI at 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Counted only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($226.75). It also miscalculated Medicare as $252.97 instead of $252.92. -us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Excluded the $8,089 ESI premium from the wage base, so taxes were applied to $9,354 instead of $17,443. Its $818 equals 8.75% of $9,354, so it also used a 1.1% SDI rate instead of 1.3%." -us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Subtracted the $8,089 ESI premium to get a $9,354 base, although all of the payroll taxes apply to the full $17,443 of gross wages. It also used a 1.1% CA SDI rate instead of the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $1,595 total implies about $261 of CA SDI, roughly a 1.5% rate. That overstates the 2026 SDI rate of 1.3%, which gives $226.75 on top of the $1,334.36 of FICA." -us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,Applied a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $809.12 equals 8.65% of $9,354. That means it subtracted the $8,089 ESI premium from wages and used a 1.0% SDI rate. The correct calculation is 7.65% FICA plus 1.3% SDI on the full $17,443." -us,scenario_023,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,Used 1.1% for CA SDI ($191.87) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,glm-5.2,llm_error,state_local_rule,False,Used the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and left out the 1.3% SDI ($226.75). It then submitted an unexplained $1,329 instead of its own $1,334.39 FICA total." -us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said California has no mandatory state payroll tax and left out the 1.3% employee SDI ($226.75). Its $1,375.97 is also not equal to 7.65% of $17,443 ($1,334.39), which shows an arithmetic error." -us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out CA SDI (1.3%, $226.75). It also made arithmetic errors: Social Security on $17,443 is $1,081.47, not $1,070.1, and its own components add to about $1,323, not $1,208." -us,scenario_023,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"Estimated CA SDI at the 2025 rate of 1.2%, for a combined 8.85% rate. With the 2026 SDI rate of 1.3%, the combined rate is 8.95% and SDI is $226.75." -us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Applied only the 7.65% FICA rate and left out California's mandatory 1.3% employee SDI ($226.75). -us,scenario_023,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,Estimated CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Used the correct 1.3% SDI rate but subtracted the $8,089 ESI premium from the wage base, applying 8.95% to $9,354. The rate applies to the full $17,443 of gross wages." -us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Counted only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($226.75). -us,scenario_023,payroll_tax,grok-4.5,llm_error,state_local_rule,False,Carried the 2025 CA SDI rate of 1.2% ($209.32) into 2026 instead of using 1.3% ($226.75). -us,scenario_023,payroll_tax,grok-4.6,llm_error,state_local_rule,False,Used 1.2% for CA SDI ($209.32) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,grok-4.7,llm_error,state_local_rule,False,Carried the 1.2% CA SDI rate forward instead of applying the 2026 rate of 1.3%. This understated SDI by about $17. -us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted the $2,778 traditional 401(k) deferral from FICA wages. Elective deferrals stay subject to Social Security, Medicare and SDI. It also used a 1.1% SDI rate instead of 1.3%." -us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"Applied the taxes to $9,354 after subtracting the $8,089 ESI premium, instead of to the full $17,443 of wages. It also used a 1.2% CA SDI rate instead of the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not submit a payroll_tax value or any explanation. The correct derivation is FICA of $1,334.36 plus CA SDI of $226.75 at 1.3%, for $1,561.12." -us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"Did not submit a payroll_tax value or any explanation. The correct derivation is FICA of $1,334.36 plus CA SDI of $226.75 at 1.3%, for $1,561.12." -us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Counted only federal FICA and left out California's mandatory employee SDI, which is 1.3% of all wages ($226.75)." -us,scenario_023,payroll_tax,ox-alpha,llm_error,state_local_rule,False,Used the 2025 CA SDI rate of 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75). -us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Correctly said CA SDI is a mandatory employee payroll tax, but used 1.1% instead of 2026's 1.3%. It then submitted the FICA-only figure of $1,334.39 and dropped SDI completely." -us,scenario_023,payroll_tax,qwen3.8-max,llm_error,other,False,"Its $261.65 is only about 1.5% of wages, so it effectively counted Medicare alone. It dropped the 6.2% Social Security tax ($1,081.44) and the 1.3% CA SDI ($226.75)." +us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 1.13% CA SDI rate ($197.11) instead of the 2026 rate of 1.3% ($226.75), so its total came to about $1,531 instead of $1,561. Its FICA components were correct." +us,scenario_023,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model counted only federal FICA ($1,334.39) and left out California's mandatory employee SDI contribution of 1.3% of wages ($226.75). That is a required employee state payroll tax under the question's definition." +us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model wrongly said California has no mandatory employee state payroll tax, so it left out the 1.3% CA SDI contribution ($226.75). It also submitted $1,335.85, which does not match its own FICA sum of $1,334.38." +us,scenario_023,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,The model counted only 7.65% FICA and left out California's mandatory employee SDI at 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75), which understated SDI by about $17." +us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an outdated 0.9% CA SDI rate ($156.99) instead of 1.3% ($226.75). It then submitted $1,467.32, which does not match its own stated sum of $1,490.38." +us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75). It also submitted $1,546, which does not match its own total of about $1,544." +us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,The model used the 2024–2025 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). Its FICA base and components were correct. +us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model wrongly assumed California has no mandatory employee payroll tax and left out CA SDI. That contribution is 1.3% of all wages in 2026 ($226.75). +us,scenario_023,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model deliberately left CA SDI out, even though the question explicitly includes mandatory employee state payroll taxes. SDI at 1.3% adds $226.75 to the FICA total of $1,334.39." +us,scenario_023,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model subtracted the $8,089 employer-sponsored insurance premiums from wages and taxed only $9,354, but the payroll base is the full $17,443 of wages. It also left out California's 1.3% SDI contribution." +us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model reduced the FICA and SDI base to $9,354 by subtracting the $8,089 ESI premiums, but payroll taxes apply to the full $17,443 of wages. It also used a 1.2% SDI rate instead of 1.3%." +us,scenario_023,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,The model used a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75). Its FICA components were correct. +us,scenario_023,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied a combined rate of 8.85%, which assumes 1.2% CA SDI. The correct combined rate is 8.95% because 2026 SDI is 1.3%, which gives about $1,561." +us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model estimated CA SDI at 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model left out California's mandatory 1.3% employee SDI contribution ($226.75). It also miscalculated Medicare as $252.97 instead of $252.92. +us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Its $818 equals 8.75% (7.65% FICA plus 1.1% SDI) of $9,354, so it removed the $8,089 ESI premiums from the payroll wage base. The base should be the full $17,443, and SDI should be 1.3%." +us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the $8,089 ESI premiums and taxed only $9,354, when payroll taxes apply to the full $17,443 of wages. It also used a 1.1% SDI rate instead of 1.3%." +us,scenario_023,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model's $1,595 implies about $261 of CA SDI, roughly 1.5% of wages. The correct 2026 SDI is 1.3%, or $226.75." +us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,The model used a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $809.12 equals 8.65% of $9,354, so it subtracted the $8,089 ESI premiums from wages and applied only a 1.0% SDI rate. The correct calculation uses the full $17,443 of wages with 1.3% SDI." +us,scenario_023,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,The model used a 1.1% CA SDI rate ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,The model used a 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"The model wrongly said California has no mandatory employee payroll tax and left out the 1.3% SDI ($226.75). It also submitted $1,329, which does not match its own FICA sum of $1,334.39." +us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model wrongly said California has no mandatory employee payroll tax and left out the 1.3% SDI. It also submitted $1,375.97, which does not equal 7.65% of $17,443 ($1,334.39)." +us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"The model left out California's 1.3% SDI contribution. It also miscalculated FICA: it computed $1,070.10 for Social Security instead of $1,081.47 and submitted $1,208 instead of about $1,334." +us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated CA SDI at 1.2% instead of the 2026 rate of 1.3%, which gave a combined rate of 8.85% instead of 8.95%." +us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model counted only 7.65% FICA and left out California's mandatory employee SDI at 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,gpt-6-luna,llm_error,thresholds_rates,False,The model estimated CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model used the correct 1.3% SDI rate but applied FICA and SDI to $9,354 after subtracting the $8,089 ESI premiums. Payroll taxes apply to the full $17,443 of wages." +us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model counted only Social Security and Medicare and left out California's mandatory employee SDI at 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,The model used a projected 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,The model used a 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,The model carried forward a 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $2,778 traditional 401(k) deferral from the FICA and SDI wage base, but elective deferrals remain subject to FICA and SDI. It also used a 1.1% SDI rate instead of 1.3%." +us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model removed the $8,089 ESI premiums and taxed only $9,354, when payroll taxes apply to the full $17,443 of wages. It also used a 1.2% SDI rate instead of 1.3%." +us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so there was no answer to grade." +us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so there was no answer to grade." +us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model counted only federal FICA and left out California's mandatory employee SDI at 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,muse-wonder,llm_error,thresholds_rates,False,"The model used a 1.2% CA SDI rate instead of the 2026 rate of 1.3%, which understated SDI by about $17." +us,scenario_023,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,The model used a 1.2% CA SDI rate ($209) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model calculated SDI at 1.1% and reached $1,526.26 in its reasoning, but submitted only the FICA amount of $1,334.39, which leaves CA SDI out entirely. The 2026 SDI rate is 1.3% ($226.75), not 1.1%." +us,scenario_023,payroll_tax,qwen3.8-max,llm_error,other,False,"Its $261.65 equals 1.5% of $17,443, which matches applying only a Medicare-like rate. That leaves out the 6.2% employee Social Security tax ($1,081.44) and the 1.3% CA SDI ($226.75)." us,scenario_023,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Treated the 'is disabled' flag as making this a SNAP elderly/disabled household, so it took an excess medical deduction and an uncapped shelter deduction. SNAP disability status requires receiving a disability-based benefit such as SSI or SSDI, which the head does not, so the shelter deduction is capped at $744 and net income is $876.51, not $345. It also submitted $3,936 even though its own calculation reached $2,328." -us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Applied the medical deduction and the uncapped excess shelter deduction (rent plus SUA) on the theory that the head is disabled, which brought net income to $0 and gave the $298 maximum. The head receives no SSI or SSDI, so the household is not SNAP-disabled: the $744 shelter cap applies, there is no medical deduction, and net income of $876.51 yields $35.20/month." -us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Applied the federal 130% FPL gross income test and treated high rent as disqualifying. California's broad-based categorical eligibility (through TANF non-cash assistance) sets the gross limit at 200% FPL, which $2,120/month passes. Rent increases the shelter deduction rather than disqualifying the household, so net income of $876.51 gives $35.20/month, not $0." -us,scenario_023,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Concluded that income exceeds SNAP limits. In fact the household passes California's 200% FPL broad-based categorical eligibility gross screen, and after the 20% earned-income deduction, the $209 standard deduction and the $744 capped shelter deduction, net income is $876.51. That is below the $1,304.17 net limit and yields $35.20/month." -us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled and applied an uncapped $14,837/year shelter deduction plus a medical deduction, but SNAP disability requires receiving SSI, SSDI or a similar benefit, so the $744/month cap applies and there is no medical deduction. It also submitted $3,768 even though its own calculation reached $2,529." -us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Applied the medical deduction and an uncapped shelter deduction because the head is 'disabled'. The head receives no disability-based benefit, so the shelter deduction is capped at $744 and net income is $876.51. It also miscomputed the earned-income deduction as $121/month instead of $290.72, then submitted $2,549, which does not match its own $1,788 result." -us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Misstated California's 200% FPL gross limit as about $2,510 and concluded the household fails, when $2,120/month is under the roughly $2,608 limit. With the $744 shelter cap, net income is $876.51, below the $1,304.17 net limit, so the benefit is $35.20/month rather than zero." -us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Assumed the disability flag removes the shelter cap and allows a medical deduction, which brought net income to $0. SNAP elderly/disabled status requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies and net income of $876.51 yields only $35.20/month." -us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Took a $73.33 medical deduction and an uncapped $1,223.90 shelter deduction for a 'disabled' head, even though the head receives no disability-based benefit that would make them SNAP-disabled. With the $744 cap and no medical deduction, net income is $876.51. The model also used the outdated $292 maximum allotment and $204 standard deduction instead of $298 and $209." -us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Applied the federal 130% FPL gross test and ignored California's broad-based categorical eligibility, which sets the gross limit at 200% FPL; the household passes at $2,120/month. Net income after the capped $744 shelter deduction is $876.51, which yields $35.20/month instead of zero." -us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Waived the shelter cap and added a medical deduction because the head is 'disabled', which gave net income of about $346. The head receives no SSI or SSDI, so the household is not SNAP-disabled: the $744 cap holds, net income is $876.51, and the benefit is $35.20/month." -us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Used an uncapped excess shelter deduction because the head is disabled, which gave net income of $445.80. SNAP disability status requires receiving a disability-based benefit, so the shelter deduction is capped at $744, giving net income of $876.51 and a benefit of $35.20/month." -us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Treated the household as SNAP-disabled and used an uncapped $1,560.94 shelter deduction. Its $747.42 medical deduction also counted the $8,089 employer-sponsored premium as an out-of-pocket medical expense. Without SSI or SSDI receipt there is no medical deduction and the shelter deduction is capped at $744, so net income is $876.51." -us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied a $98.33 excess medical deduction and an uncapped $1,238.90 shelter deduction for a 'disabled' head. SNAP disability requires receiving a disability-based benefit, so the $744 shelter cap applies with no medical deduction, giving net income of $876.51 and $35.20/month." -us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Waived the gross test and used uncapped shelter and medical deductions because the head is disabled, which gave net income of $344.29. The head is not SNAP-disabled because they receive no SSI or SSDI, so the $744 cap applies, net income is $876.51, and the benefit is $35.20/month." -us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Applied a medical deduction and an uncapped shelter deduction (rent plus a $595 SUA) on the theory that the head is disabled, which brought net income to $0, and used a $291 maximum allotment. The household is not SNAP-disabled without SSI or SSDI receipt: the shelter deduction is capped at $744, net income is $876.51, and the benefit is $298 − $262.80 = $35.20/month." -us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Gave no derivation. Its $3,492 equals 12 × a $291 maximum allotment, which implies zero net income from uncapped shelter deductions. The correct calculation caps the shelter deduction at $744 because the head receives no disability-based benefit, giving net income of $876.51 and $35.20/month." -us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Claimed that disability makes the shelter deduction uncapped and allows a medical deduction, which brought net income to zero. SNAP's disabled-member definition requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies and net income of $876.51 yields $35.20/month." -us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Used uncapped shelter and medical deductions for a 'disabled' head, which gave net income of about $300. Without receipt of a disability-based benefit the household is not SNAP-disabled, so the shelter deduction is capped at $744 and net income is $876.51." -us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted that income or resources exceed the limits. The household passes California's 200% FPL gross screen, its $130 in assets is far below any limit, and net income after the $744 capped shelter deduction is $876.51, below the $1,304.17 net limit, which yields $35.20/month." -us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Applied an uncapped excess shelter deduction because the head is disabled. SNAP's disability definition requires receiving SSI, SSDI or a similar benefit, so the deduction is capped at $744 and net income is $876.51, which yields $35.20/month rather than about $165." -us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the household as disabled for SNAP and used an uncapped shelter deduction, which brought net income to $0 and gave the $298 maximum. The head receives no disability-based benefit, so the $744 cap applies and net income of $876.51 gives $35.20/month." -us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Applied uncapped shelter and medical deductions on the basis of the disability flag. SNAP-disabled status requires receiving a disability-based benefit, which the head does not, so the shelter deduction is capped at $744, there is no medical deduction, and net income is $876.51." -us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Exempted the household from the gross test and took a $1,180 medical deduction and an uncapped $14,836.80 shelter deduction because the head is disabled. Without SSI or SSDI receipt the shelter deduction is capped at $744/month with no medical deduction. It also used the outdated $292 maximum allotment ($3,504/year) instead of $298." -us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Subtracted a monthly standard deduction from annual income and compared 30% of annual net income to a monthly maximum allotment, which mixes periods and wrongly zeroes the benefit. It also never applied the shelter deduction, then submitted an unsupported $871. The correct monthly calculation gives net income of $876.51 and $35.20/month." -us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Concluded that income and assets are too high. Gross income of $2,120/month passes California's 200% FPL broad-based categorical eligibility screen, $130 in assets is negligible, and net income after the $744 capped shelter deduction is $876.51, which yields $35.20/month." -us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Declined to compute SNAP even though the prompt says to assume take-up. The household is eligible through California's 200% FPL broad-based categorical eligibility, and net income of $876.51 after the $744 capped shelter deduction yields $35.20/month." -us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Used medical and uncapped excess shelter deductions for a 'disabled' household, which brought net income to $0. The head receives no SSI or SSDI, so the household is not SNAP-disabled: the shelter deduction is capped at $744, net income is $876.51, and the benefit is $35.20/month." -us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied the medical deduction and the uncapped shelter deduction because the head is disabled, which produced the maximum allotment. SNAP disability requires receiving a disability-based benefit, so the $744 shelter cap applies and net income of $876.51 yields $35.20/month." -us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Applied an uncapped 'disability shelter' deduction and a medical deduction for the head. Without SSI or SSDI receipt the household is not SNAP elderly/disabled, so the shelter deduction is capped at $744, there is no medical deduction, and net income is $876.51." -us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Concluded that wages plus the 403(b) distribution leave no benefit. Gross income passes California's 200% FPL screen, and net income after the 20% earned-income deduction, the $209 standard deduction and the $744 capped shelter deduction is $876.51, which leaves a $35.20/month benefit." -us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Applied California's standard medical deduction and an uncapped shelter deduction because the head is disabled, which gave net income of $205.80. The head receives no disability-based benefit, so neither the medical deduction nor the uncapped shelter deduction is available. The $744 cap gives net income of $876.51 and $35.20/month." -us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled and used rent and medical deductions to reach the maximum $298. SNAP disability requires receiving SSI, SSDI or a similar benefit, so the $744 shelter cap applies, there is no medical deduction, and net income of $876.51 yields $35.20/month." -us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Applied an uncapped shelter deduction because of the disability flag, which brought countable income to zero. The head is not SNAP-disabled without receiving a disability-based benefit, so the shelter deduction is capped at $744 and net income is $876.51." -us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Took an $880 disability-related medical deduction and an uncapped shelter deduction, which gave annual net income of $3,849.60. Because the head receives no SSI or SSDI, neither deduction applies as computed: the shelter deduction is capped at $744/month, net income is $876.51/month, and the benefit is $35.20/month." -us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed that income and assets exceed SNAP limits. Assets of $130 are trivial, gross income passes California's 200% FPL broad-based categorical eligibility screen, and net income after the $744 capped shelter deduction is $876.51, below the $1,304.17 net limit, which yields $35.20/month." -us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Applied a $1,180 excess medical deduction and an uncapped $14,837 shelter deduction for a 'disabled' household. SNAP disability requires receiving a disability-based benefit, so the shelter deduction is capped at $744/month with no medical deduction. It also used the outdated $292 maximum allotment ($3,504/year)." -us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Correctly passed the 200% FPL broad-based categorical eligibility test but used medical and uncapped shelter deductions, which brought net income to $0, and an outdated $292 maximum allotment. The head is not SNAP-disabled without SSI or SSDI receipt, so the $744 cap applies, net income is $876.51, and the benefit is $35.20/month." -us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Applied an excess medical deduction on $1,600 of costs and an uncapped shelter deduction, which gave net income of about $283. Without receipt of a disability-based benefit the household is not SNAP-disabled: the $744 shelter cap applies, net income is $876.51, and the benefit is $35.20/month." -us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Left the $8,000 taxable 403(b) distribution out of countable unearned income and applied uncapped shelter and medical deductions for a 'disabled' head. The correct gross income is $2,120.25/month, and with the $744 shelter cap and no medical deduction, net income is $876.51." -us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"Used the medical deduction and an uncapped shelter deduction for the disabled head, which gave net income of about $300. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and net income is $876.51, which yields $35.20/month." -us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"Exempted the household from the gross test and applied uncapped shelter and medical deductions because of the disability flag, which brought net income to $0, and used a $292 maximum. SNAP disability requires receiving a disability-based benefit, so the $744 cap applies and net income of $876.51 yields $35.20/month." -us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"Returned no answer or explanation for snap, so there was no value to score against the $461.34 reference." -us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted the traditional 401(k) and IRA contributions to get $22,535 of gross income, then applied the federal 130% FPL gross test. California's broad-based categorical eligibility uses a 200% FPL gross limit, which the $25,443 gross income passes, and net income of $876.51 yields $35.20/month." -us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Correctly applied the $744 shelter cap but used the FY2025 $204 standard deduction instead of $209. It also applied FY2026 parameters to all 12 months, missing the October–December 2026 increase to a $304.68 maximum allotment with net income of $855, which raises those three months to $48.18 each." -us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Left the $8,000 403(b) distribution out of countable income, applied no shelter deduction on $2,000/month rent, and then set the benefit equal to the maximum allotment despite positive net income. The correct net income is $876.51, which gives $298 − $262.80 = $35.20/month." -us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that income and resources are too high. Gross income passes California's 200% FPL broad-based categorical eligibility screen, assets are only $130, and net income after the $744 capped shelter deduction is $876.51, below the net limit, which yields $35.20/month." +us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"The model treated the 'is disabled' flag as making this an elderly/disabled SNAP household, so it took an excess medical deduction and an uncapped shelter deduction, getting net income of $345. The head receives no disability-based benefit, so the $744 shelter cap applies, there is no medical deduction, and net income is about $876. The submitted $3,936 also contradicts the model's own $2,328 calculation." +us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model applied a disabled-household medical deduction and an uncapped shelter deduction (rent plus SUA), driving net income to $0 and giving the $298 maximum allotment. The head receives no disability-based benefit, so the household is not SNAP elderly/disabled. Excess shelter is capped at $744/month, net income is $876.51, and the benefit is only about $35/month." +us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model applied the federal 130% FPL gross income test and a $2,500 asset limit. California's broad-based categorical eligibility (through TANF non-cash assistance) uses a 200% FPL gross limit, which $2,120/month passes. Net income after the capped shelter deduction is $876.51, below the net limit, so the household gets a small positive benefit." +us,scenario_023,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model said income exceeds SNAP limits, but the household passes California's 200% FPL gross test under broad-based categorical eligibility. Its net income of $876.51 (after the earned-income, standard and $744 capped shelter deductions) is 67% of FPL, which gives about $35–48/month in the frozen reference's projected schedule; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled, deducting $1,180 of medical expenses and an uncapped $14,837 annual shelter amount. The head receives no disability benefit, so there is no medical deduction and shelter is capped at $744/month. The model also replaced its own $2,529 result with an unsupported $3,768." +us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model took a disabled-household medical deduction and an uncapped shelter deduction, even though the head receives no disability-based benefit and the $744 cap applies. It also miscomputed the 20% earned-income deduction as $121/month instead of $291, and it dropped its own $1,788 result for an unsupported $2,549." +us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"The model correctly applied the shelter cap but wrongly concluded the household fails SNAP. Monthly gross of $2,120 passes California's 200% FPL BBCE test, and capped net income of $876.51 passes the net-income test, whose monthly limit is rounded up to a whole dollar. The frozen reference gives $35.20/month for January–September; under c_snap_hold_fy2026 and the SNAP rounding fixes, this capped reading gives $34/month ($408/year), so the benefit remains positive." +us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model assumed the disability flag gives uncapped medical and shelter deductions, reducing net income to $0 and producing the $298 maximum. For SNAP, disability requires receipt of a disability-based benefit, which the head lacks. Shelter is therefore capped at $744, there is no medical deduction, and net income is $876.51." +us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"The model applied a $73/month excess medical deduction and an uncapped $1,224 shelter deduction because the head is disabled. The head receives no SSI/SSDI-type benefit, so the household gets the $744 capped shelter deduction and no medical deduction, which gives net income of $876.51 instead of $328." +us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model disqualified the household with the federal 130% FPL gross income test and ignored California's broad-based categorical eligibility, which raises the gross limit to 200% FPL. It also never applied the excess shelter deduction, which brings net income down to $876.51 and leaves a positive benefit." +us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model used a disabled-household medical deduction and an uncapped excess shelter deduction, getting net income of about $346. The head receives no disability-based benefit, so the $744 shelter cap applies, there is no medical deduction, and net income is $876.51, giving about $35/month instead of $194." +us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model left the excess shelter deduction uncapped because the head is disabled. That exemption requires the household to meet the SNAP elderly/disabled definition, which needs receipt of a disability benefit. With the $744 cap, net income is $876.51 rather than about $446." +us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated the household as SNAP-disabled and took a $747/month medical deduction (wrongly including the $8,089 employer-paid insurance premiums) plus an uncapped $1,561 shelter deduction. The head is not SNAP-disabled, so there is no medical deduction and shelter is capped at $744, leaving $876.51 of net income." +us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied a $98 excess medical deduction and an uncapped $1,239 shelter deduction for a disabled household. The head receives no disability-based benefit, so neither applies beyond the $744 capped shelter deduction, and net income is $876.51, not $283." +us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model treated the disability flag as qualifying for the medical deduction and an uncapped shelter deduction, getting net income of $344. Under the SNAP disability definition, which requires receiving a disability benefit, the shelter deduction is capped at $744 and there is no medical deduction, so net income is $876.51." +us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model applied an uncapped shelter deduction (rent plus a $595 SUA) and a medical deduction for a disabled head, reducing net income to $0. The head is not SNAP-disabled because they receive no disability benefit, so shelter is capped at $744 and net income is $876.51, which yields $35–48/month in the frozen reference's projected schedule; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"The model gave no reasoning, but $3,492 is 12 × a $291 maximum allotment, which implies net income of zero from uncapped disability deductions. The head is not SNAP-disabled, so the $744 shelter cap leaves net income of $876.51 and about $35–48/month in benefits in the frozen reference's projected schedule; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model assumed the disability flag removes the shelter cap and allows medical deductions, zeroing net income. SNAP's disabled status requires receiving a disability-based benefit, so the shelter deduction is capped at $744, net income is $876.51, and the frozen reference gives $35.20/month for January–September; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model used uncapped shelter and medical deductions for a disabled head, getting net income of about $300. The head does not meet SNAP's benefit-receipt disability definition, so the $744 shelter cap applies with no medical deduction, which puts net income at $876.51." +us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"The model claimed income or resources exceed SNAP limits. The household passes California's 200% FPL BBCE gross test, has $130 in assets, and its net income of $876.51 after the capped shelter deduction is 67% of FPL, which gives a positive benefit." +us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model applied uncapped shelter expenses for a disabled individual. The head receives no disability-based benefit, so SNAP caps excess shelter at $744/month, net income is $876.51, and the benefit is about $35/month rather than $165." +us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled with an uncapped shelter deduction, reducing net income to $0 and awarding the maximum. Without receipt of a disability benefit, the shelter deduction is capped at $744, net income is $876.51, and the frozen reference's allotment is $35.20/month for January–September, followed by a projected FY2027 increase; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"The model applied an uncapped excess shelter deduction and medical deductions for a disabled head. The head is not SNAP-disabled (no disability-based benefit), so shelter is capped at $744 with no medical deduction, which leaves net income of $876.51." +us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"The model deducted $1,180 of excess medical expenses and an uncapped $14,837 annual shelter amount because the head is disabled. The head receives no disability benefit, so annual shelter is capped at $8,928 ($744 × 12) with no medical deduction, and net income is about $10,518 a year rather than $3,490." +us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"The model omitted the excess shelter deduction entirely and compared 30% of annual net income to a monthly allotment, which led it to conclude the benefit was zero. It then submitted an unsupported $871. Applying the $744 capped shelter deduction gives monthly net income of $876.51 and the frozen reference's $35.20/month for January–September; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model asserted that income and assets are too high without computing anything. Gross income of $2,120/month passes California's 200% FPL BBCE limit, assets are $130, and net income after the capped $744 shelter deduction is $876.51, so the benefit is positive." +us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model set SNAP to zero because the inputs did not explicitly state eligibility, ignoring the instruction to assume take-up. A full computation shows the household is categorically eligible through California BBCE and has net income of $876.51, which yields a positive allotment." +us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"The model applied medical and uncapped excess-shelter deductions for a disabled household, reducing net income to $0 and awarding $298/month. The head receives no disability-based benefit, so the shelter deduction is capped at $744 with no medical deduction, which leaves net income of $876.51." +us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model used the medical deduction and an uncapped shelter deduction for a disabled one-person household to reach the maximum allotment. SNAP disability status requires receiving a disability benefit, so the $744 shelter cap applies and the frozen reference gives $35.20–$48.18/month using projected FY2027 parameters for October–December; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model correctly cleared the student rule, then applied a disability medical deduction and an uncapped shelter deduction. The head is not SNAP-disabled, so excess shelter is capped at $744 with no medical deduction, and net income of $876.51 gives about $35/month, not $202." +us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model concluded there was no benefit after deductions. The standard ($209), 20% earned-income ($291) and capped shelter ($744) deductions bring net income to $876.51, and the rounded expected contribution is $264, below the $298 maximum, giving $34/month ($408/year) on this capped reading under c_snap_hold_fy2026 and the SNAP rounding fixes." +us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model applied California's $150 standard medical deduction and an uncapped shelter deduction for disability. The head receives no disability-based benefit, so neither applies beyond the $744 capped shelter deduction, and net income is $876.51 rather than $205.80." +us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model treated the disability flag as unlocking rent and medical deductions that zero out net income. Under SNAP's benefit-receipt disability definition the shelter deduction is capped at $744 and there is no medical deduction, so net income is $876.51 and the benefit is far below the maximum." +us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model used an uncapped shelter deduction for disability to drive countable income to zero. The head receives no disability benefit, so the shelter deduction is capped at $744, net income is $876.51, and the frozen reference gives $35.20/month for January–September, followed by a projected FY2027 increase; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model deducted $880 of disability-related medical expenses and an uncapped shelter amount, getting annual net income of $3,850. The head is not SNAP-disabled, so shelter is capped at $744/month with no medical deduction, which puts annual net income at about $10,518." +us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"The model said income and assets exceed SNAP limits. Gross income passes California's 200% FPL BBCE test, $130 in assets is negligible, and net income after the capped shelter deduction is $876.51, below 100% FPL, so the benefit is positive." +us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"The model applied a $1,180 excess medical deduction and an uncapped $14,837 annual shelter deduction for a disabled household. The head receives no disability-based benefit, so the $744/month shelter cap applies with no medical deduction, and net income is about $876.51/month." +us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"The model treated the household as disabled, with excess medical and uncapped shelter deductions, reducing net income to $0 and awarding the maximum. Under SNAP's benefit-receipt disability definition the shelter deduction is capped at $744, net income is $876.51, and the frozen reference gives $35.20/month for January–September; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"The model applied an excess medical deduction on $1,600 of costs and an uncapped shelter deduction, getting net income of $283. The head is not SNAP-disabled, so there is no medical deduction and shelter is capped at $744, which gives net income of $876.51." +us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model omitted the $8,000 403(b) distribution from countable unearned income. It also applied a disability medical deduction and an uncapped shelter deduction, even though the head receives no disability benefit and the $744 cap applies. Net income is $876.51, not $0." +us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"The model used a medical deduction for the disabled head and an uncapped shelter deduction, estimating net income of $300. The head is not SNAP-disabled, so shelter is capped at $744 with no medical deduction, and net income is $876.51, which gives about $35/month." +us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model exempted the household from the shelter cap and allowed medical deductions because the head is disabled, zeroing net income. SNAP disability requires receiving a disability-based benefit, so the $744 shelter cap applies, net income is $876.51, and the frozen reference gives $35.20–$48.18/month using projected FY2027 parameters for October–December; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, so there is no answer to evaluate." +us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"The model applied the federal 130% FPL gross income test and ignored California's broad-based categorical eligibility, which sets a 200% FPL gross limit that this household passes. It also misstated gross income as $22,535 instead of $25,443." +us,scenario_023,snap,muse-wonder,prompt_ambiguity,age_disability,False,"The model treated the household as SNAP-disabled with medical and uncapped shelter deductions, reducing net income to $0. The head receives no disability benefit, so shelter is capped at $744, net income is $876.51, and the frozen reference gives $35.20/month for January–September, followed by a projected FY2027 increase; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading." +us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"The model correctly capped the shelter deduction at $744 and applied FY2026 parameters to all 12 months, as c_snap_hold_fy2026 requires, but used a $204 standard deduction instead of $209. The frozen reference reaches $461.34 using the engine's projected FY2027 $304.68 maximum and $256.50 contribution ($48.18/month) for October–December; on the capped reading, the current engine with the convention and SNAP rounding fixes gives $408 ($34 × 12), compared with the model's $402." +us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model omitted the $8,000 403(b) distribution from income and ignored the excess shelter deduction. It then set the benefit equal to the maximum allotment without subtracting 30% of its own $944 net income. The frozen reference gives net income of $876.51 and $35.20/month for January–September; under c_snap_hold_fy2026 and the SNAP rounding fixes, this capped reading gives $34/month ($408/year)." +us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model asserted that income and resources are too high under federal rules. California's BBCE 200% FPL gross limit is met, assets are $130, and net income after the capped shelter deduction is $876.51 (67% of FPL), so SNAP is positive." us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the narrative phrase “is disabled” as establishing SSI disability eligibility, despite the prompt directing that unlisted program-status inputs are false. The head fails the SSI aged, blind, or disabled gate, so applying earned-income exclusions, counting the 403(b) distribution, and subtracting income from the federal-plus-California payment standard was inapplicable." us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." -us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model built AGI (~$22,535) and the standard deduction (~$5,700) correctly but put the tax at about $200 instead of $225.78 (1% of $11,079 plus 2% of the remaining $5,749). It then declared the $213 of credits a full offset, even though $12.78 remains." -us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a projected 2026 standard deduction of ~$5,880 instead of the 2025 amount of $5,706. That cut taxable income to ~$16,655 and the tax to ~$222, so it got $9 instead of $225.78 − $213 = $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used wages alone, so it left out the $8,000 taxable 403(b) distribution, skipped the 401(k) and IRA adjustments, and used an outdated $5,202 standard deduction. It then subtracted CalEITC as if it were nonrefundable; CalEITC is a refundable credit and does not belong in this output." -us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed California taxable income is zero, but CA AGI of $22,534 (including the $8,000 403(b) distribution) far exceeds the $5,706 standard deduction. That leaves $16,828 taxable and $225.78 of tax before the $213 of credits." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the older $5,540 standard deduction, put the tax at ~$170 (the 1%/2% schedule on its own ~$17,000 gives ~$240), and subtracted only a ~$149 exemption credit plus an undefined disability adjustment. It concluded $0 instead of $225.78 − $153 − $60 = $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own figures (~$233 tax minus a ~$149 exemption credit) leave a positive balance, yet it called the remaining tax effectively $0. It also never applied the $60 renter's credit. The correct net is $225.78 − $213 = $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model understated the tax as ≈$200 (it is $225.78) and asserted that a ~$150 exemption credit fully offsets it. The $153 exemption credit plus the $60 renter's credit total $213, which leaves $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model projected 2026 parameters: a ~$5,880 standard deduction and a ~$158 exemption credit. The applicable values are the 2025 ones, $5,706 and $153, which give $225.78 of tax and $12.78 after credits, not $1." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model invented a second $144 exemption credit for disability. California's personal exemption credits are for the taxpayer, blind status, and dependents, with no disability credit. It also used the 2024 standard deduction and brackets ($5,540 and $10,756) and omitted the renter's credit. With the correct $153 + $60 credits, $12.78 remains." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model guessed $250–$300 of tax, then claimed the ~$149 exemption credit exceeds it, which contradicts its own numbers. The correct $225.78 tax minus the $153 exemption credit and the $60 renter's credit leaves $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"The model computed the tax (~$226) and the $153 exemption credit correctly but left out the $60 nonrefundable renter's credit. This filer qualifies for it: they pay $24,000 in rent and AGI is under the single-filer limit. It also rounded $73 up to $75." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model's CA AGI of $14,446 leaves out the $8,000 taxable 403(b) distribution, which is included in California AGI. Including it gives $22,534 AGI, $16,828 taxable income, and $225.78 of tax, which the credits do not fully offset." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a CA AGI of $14,446, which omits the $8,000 taxable 403(b) distribution, together with an outdated $5,363 standard deduction. It therefore taxed only ~$9,083 at 1%, when the correct taxable income is $16,828.34 with $225.78 of tax." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model projected 2026 indexing: a $5,876 standard deduction, a $11,044 bracket threshold, and a $158 exemption credit. The applicable values are the 2025 ones ($5,706, $11,079, and $153), which give $12.78 rather than $4.74." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model left out the $130 traditional IRA deduction (AGI $22,665 instead of $22,534) and used the 2024 standard deduction ($5,540) and 1% bracket top ($10,756) instead of 2025's $5,706 and $11,079. That inflated the tax to $234.94 and the result to $20.94." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model approximated the standard deduction (~$5,800 vs $5,706), the tax (~$230 vs $225.78), and the exemption credit (~$155 vs $153), then rounded. The exact 2025-parameter calculation gives $225.78 − $213 = $12.78, not $15." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model claimed the single standard deduction exceeds AGI, but the $5,706 deduction is far below the $22,534 CA AGI. That AGI includes wages net of the 401(k) deferral plus the $8,000 403(b) distribution." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"The model asserted that the personal exemption credit fully offsets the tax. The $153 credit covers only part of the $225.78 tax, and even with the $60 renter's credit $12.78 remains." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's taxable income of $8,846 leaves out the $8,000 taxable 403(b) distribution. With it included, taxable income is $16,828.34 and the tax of $225.78 exceeds the $213 of nonrefundable credits." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"With no derivation, the model's $0 implies it treated the tax as fully offset. The correct calculation is $22,534 AGI minus the $5,706 standard deduction, giving $225.78 of tax, less the $153 exemption and $60 renter's credits, for $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,The $218 answer is roughly the tax before credits: the model never subtracted the $153 personal exemption credit or the $60 renter's credit. Both are nonrefundable and reduce the $225.78 tax to $12.78. -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model claimed the standard deduction and exemption credit eliminate the tax. Taxable income is $16,828.34 with $225.78 of tax, and the $153 exemption plus $60 renter's credit leave $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied the right structure (1%/2% brackets, exemption credit, $60 renter's credit) but used approximate parameters instead of the 2025 values. The exact values ($5,706 deduction, $11,079 bracket, $153 credit) give $12.78, not $17." -us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated California's personal exemption as a $138 deduction from income instead of a nonrefundable credit (2025: $153). It said no nonrefundable credits apply, which skipped both that credit and the $60 renter's credit. It also used the outdated 2023 $5,363 deduction and $10,234 bracket." -us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used $25,443 as CA AGI, failing to exclude the $2,778 traditional 401(k) deferral and deduct the $130 IRA contribution, both of which California conforms to. It also left out the $153 nonrefundable personal exemption credit and subtracted only the $60 renter's credit." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model asserted the deduction and credits wipe out the tax without computing it. The $225.78 tax on $16,828 of taxable income exceeds the $213 of exemption and renter's credits by $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The $72 answer equals the $225.78 tax minus only the $153 exemption credit, so the $60 nonrefundable renter's credit was never applied. The American Opportunity Credit the model cites is a federal credit and does not reduce California tax." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model added a 2.5% California early-distribution additional tax ($200) that is not part of this output. It also claimed the regular tax is fully eliminated, when $225.78 − $213 leaves $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model claimed the exemption and renter's credits fully offset the tax. Together they are $213, against $225.78 of tax on $16,828.34 of taxable income, which leaves $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used a projected 2026 standard deduction and indexed parameters instead of the 2025 values ($5,706 deduction, $11,079 bracket, $153 exemption credit). This understated the tax relative to the credits and gave $4 instead of $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model claimed the exemption and renter's credits fully offset the tax. The $153 + $60 credits total $213, below the $225.78 tax, so $12.78 remains." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model applied an indexed 2026 standard deduction and exemption credit instead of the 2025 amounts ($5,706 and $153). That gave $4 rather than $225.78 − $213 = $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The model rounded its estimate to a whole $13. The exact calculation, $225.78 of tax less $153 and $60 of nonrefundable credits, is $12.78, with no rounding to whole dollars." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model used projected 2026 indexed values for the standard deduction, brackets, and exemption credit instead of the 2025 amounts ($5,706, $11,079, and $153). That pushed the result to $3 instead of $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model's AGI of $14,446 leaves out the $8,000 taxable 403(b) distribution. California AGI is $22,534, which produces $225.78 of tax that the exemption credit does not eliminate." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed zero tax before credits, but the $5,706 standard deduction leaves $16,828.34 taxable out of $22,534 of CA AGI. That produces $225.78 of tax, and $12.78 remains after the $213 of credits." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model used an inflated 2026 exemption credit (~$158 instead of $153) and projected bracket thresholds instead of the 2025 values. That gave $9 instead of $12.78. -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model counted the $8,089 of employer-sponsored insurance premiums as an itemized medical deduction. The filer's deductible medical costs ($300 of premiums plus $500 of other medical expenses) fall below the 7.5%-of-AGI floor (~$1,690), so the $5,706 standard deduction applies and $12.78 of tax remains." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model projected 2026 parameters ($5,843 standard deduction, $11,345 bracket, $157 exemption credit) instead of the 2025 amounts ($5,706, $11,079, and $153). That reduced the tax to ~$220 and the result to $3 instead of $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,The model subtracted only the $60 renter's credit and never applied the $153 nonrefundable personal exemption credit that every single filer receives. That left $169 instead of $225.78 − $213 = $12.78. -us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's AGI of ~$14,446 leaves out the $8,000 taxable 403(b) distribution. That understated taxable income at ~$8,720 instead of $16,828.34, so the tax looked smaller than the exemption credit." -us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $12.78 reference." -us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $12.78 reference." -us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model added back the traditional 401(k) deferral and IRA deduction, but California conforms to both, so CA AGI is $22,534, not $25,443. It also said no nonrefundable credits apply, which skipped the $153 personal exemption credit and the $60 renter's credit." -us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model paired the 2024 standard deduction ($5,540) and exemption credit (~$149) with a 2025-era bracket threshold. The 2025 values are a $5,706 deduction and a $153 credit, which give $225.78 − $213 = $12.78 rather than $20." -us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model invented a $144 'disabled' exemption credit that California does not provide, and it used $25,443 of AGI without the 401(k) and IRA adjustments. It then labeled the renter's credit refundable, when California's renter's credit is nonrefundable and is part of the $213 that leaves $12.78." -us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model asserted zero tax without computing it. $16,828.34 of taxable income produces $225.78 of tax, and the $153 exemption plus $60 renter's credits leave $12.78." -us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It cut CalEITC earned income to ~$14,665 by subtracting the 401(k) deferral, but CalEITC earned income here is the full $17,443 of wages. It also left out that the phase-out runs on the higher AGI of $22,534. Its $402 is above the childless maximum of about $300, while the AGI-based phase-out gives $95.15." -us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It ran the childless phase-out on earned income of $17,443 and got ~$180. It missed that when AGI ($22,534, including the $8,000 403(b) distribution) is higher than earnings, the CalEITC phase-out uses AGI, which lowers the credit to $95.15." -us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It ruled the filer out of CalEITC for having no dependents. CalEITC is open to childless filers aged 18+ with earnings under ~$32k, so this 28-year-old with $17,443 of wages qualifies for $95.15." -us,scenario_023,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It said CalEITC does not apply without qualifying children. California pays a refundable CalEITC to childless workers aged 18+, which for this filer is $95.15 after the AGI-based phase-out." -us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It first put the childless CalEITC at ~$0-$50, then raised it to $358 by citing unnamed 'interactions' with no rule behind them. Applying the CalEITC phase-out to the higher AGI of $22,534 gives a single credit of $95.15, and no other California refundable credit applies." -us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Its $1,208 is about four times the childless CalEITC maximum (~$300) and matches a with-children schedule. The childless schedule, phased out on the higher AGI of $22,534, gives $95.15." -us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated the childless CalEITC as fully phased out below $17,443, as if the old low-income cap still applied. Since the extension, the CalEITC phases out gradually to about $32k, so at AGI $22,534 a $95.15 credit is left." -us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"It correctly used AGI of about $22.5k as the phase-out measure, but its estimated CalEITC schedule was off and gave $115 instead of $95.15." -us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It counted the $8,000 taxable 403(b) distribution as investment income and said it broke the CalEITC investment-income limit. Retirement-plan distributions are not investment income for the EITC or CalEITC; they only raise AGI, which lowers the credit to $95.15." -us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It gave a near-maximum childless CalEITC of $285 and applied no phase-out. At AGI of $22,534, well into the phase-out range, the credit is $95.15." -us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It estimated the credit on the phase-out slope using earned income of $17,443 alone. The phase-out uses the higher AGI of $22,534, which brings the CalEITC down to $95.15." -us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It used made-up figures of $6,576 earned income and $14,446 AGI, then applied a near-maximum $300 credit. Earned income is $17,443 and AGI is $22,534, and the phase-out on AGI leaves $95.15." -us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the old rule that CalEITC equals 85% of the federal EITC. The current CalEITC has its own schedule and does not depend on the federal credit. The federal childless EITC is also zero at this AGI, so its $503.06 base was wrong as well; the CalEITC schedule phased out on AGI gives $95.15." -us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It said no CalEITC applies. A childless 28-year-old with $17,443 of wages and AGI of $22,534 is within the CalEITC's ~$32k limit and gets $95.15." -us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It used an invented single linear phase-out from $15,000 to $33,000 off a $300 maximum, and a miscalculated AGI of $22,665. The actual CalEITC schedule, applied to AGI of $22,534, gives $95.15, not $172.25." -us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It correctly ran the CalEITC phase-out on the higher AGI of $22,535 but rounded to $100 instead of computing the exact 2026 schedule, which gives $95.15." -us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It said the filer's income was too high for any California refundable credit. The childless CalEITC runs to about $32k, so at AGI of $22,534 the filer still gets $95.15." -us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Its $148 matches a phase-out run on W-2 earnings of $17,443. CalEITC uses the greater of earnings or AGI, and the higher AGI of $22,534 gives $95.15." -us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Its $180 matches running the childless CalEITC phase-out on earned income of $17,443. The phase-out uses the higher AGI of $22,534, which gives $95.15." -us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,It returned $0 with no reason given and missed that this childless filer aged 18+ with earnings under ~$32k qualifies for a refundable CalEITC of $95.15. -us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It said the filer's income exceeded the CalEITC limit. The 2026 childless CalEITC runs to roughly $32k, so AGI of $22,534 still yields $95.15." -us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Its $195 matches a phase-out on wages of $17,443, or no phase-out at all. Running the phase-out on the higher AGI of $22,534 gives $95.15." -us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It correctly found a phased-out childless CalEITC but estimated the 2026 schedule instead of computing it. At AGI of $22,534 the credit is $95.15, not $92." -us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It thought the childless CalEITC was fully gone before AGI of $22,535. The CalEITC's final phase-out runs to about $32k, so a $95.15 credit remains at this AGI." -us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"It said the filer failed a childless age test. The CalEITC minimum age is 18 with no upper limit, so this 28-year-old qualifies and gets $95.15 after the AGI-based phase-out." -us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no California refundable credit and missed the childless CalEITC, which this filer qualifies for with wages of $17,443 and AGI under ~$32k. The credit is $95.15." -us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It found no California refundable credit and missed that the CalEITC covers childless workers aged 18+, which gives this filer $95.15." -us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It placed the filer in the 2026 CalEITC phase-out range but estimated the schedule loosely. Running the phase-out on AGI of $22,534 gives $95.15, not $114." -us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It computed the childless CalEITC on earned income of $17,443 alone. Because AGI ($22,534) is higher, the phase-out uses AGI, which lowers the credit to $95.15." -us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It figured the CalEITC at $17,443 of earned income. The phase-out uses the higher AGI of $22,534, which gives $95.15 instead of $160." -us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It produced no California refundable credit and missed that the childless CalEITC covers wages of $17,443 with AGI under ~$32k, giving $95.15." -us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It recognized a phased-out childless CalEITC but phased it out too far. At AGI of $22,534 the credit is $95.15, not $25." -us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It correctly ran the phase-out on the higher AGI but estimated the 2026 schedule, overshooting to $106. The exact CalEITC at AGI of $22,534 is $95.15." -us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It said a single childless adult at this income gets no CalEITC. The childless CalEITC phases out gradually to about $32k, so at AGI of $22,534 the filer gets $95.15." -us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"It applied a childless phase-out that was too steep and got $28. The 2026 CalEITC schedule at AGI of $22,534 leaves $95.15." -us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It found no qualifying California refundable credit and missed the childless CalEITC, which this filer earns at $95.15 with AGI of $22,534." -us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"It made CalEITC depend on federal EITC eligibility. CalEITC has its own schedule running to about $32k, so it pays $95.15 here even though the federal childless EITC is zero." -us,scenario_023,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"It reasoned that a zero federal EITC means a zero CalEITC. The CalEITC is computed separately from the federal credit, with its own ~$32k limit, and gives $95.15 at AGI of $22,534." -us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It correctly ran the phase-out on AGI of $22,535 but estimated the 2026 CalEITC schedule and got $120. The exact credit is $95.15." -us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"It gave about the childless CalEITC maximum ($300) and applied no phase-out. With AGI of $22,534 well into the phase-out, the credit drops to $95.15." -us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_refundable_credits, so there was nothing to score against the $95.15 CalEITC." -us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_refundable_credits, so there was nothing to score against the $95.15 CalEITC." -us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It said the CalEITC is nonrefundable for childless filers. The CalEITC is fully refundable for all eligible filers, including childless ones aged 18+, and is $95.15 here." -us,scenario_023,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It said earnings of $14,665 were far above the childless phase-out ceiling. The 2026 CalEITC ceiling is about $32k, and on the correct measures ($17,443 earnings, $22,534 AGI) the credit is $95.15." -us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It added the $60 renter's credit, which is nonrefundable and does not count. It also overstated the CalEITC at $251, which even contradicts its own ~$200 estimate. The only refundable credit is the CalEITC, phased out on AGI of $22,534 to $95.15." -us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It said no California refundable credit applies and missed the childless CalEITC, which this 28-year-old earner qualifies for at $95.15." +us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It got AGI and the standard deduction roughly right and identified both credits. It then put the tax at about $200, when 1% of $11,079 plus 2% of the remaining $5,749.34 is $225.78. Because of that, it wrongly concluded that the $213 in credits fully offset the tax, when $12.78 remains." +us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a projected 2026 standard deduction of about $5,880 instead of the $5,706 that applies. That understated taxable income and gave about $222 of tax rather than $225.78, so it reported $9 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It treated CalEITC as a nonrefundable credit that wipes out the tax. CalEITC is refundable, so it is excluded from this output. It also left the $8,000 403(b) distribution and the 401(k)/IRA adjustments out of AGI, used a $5,202 standard deduction, and never applied the $153 exemption credit or the $60 renter's credit." +us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It said California taxable income was zero. AGI is $22,534.34, which includes the $8,000 taxable 403(b) distribution, so after the $5,706 standard deduction $16,828.34 is taxable and the tax is $225.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It estimated the tax on about $17,000 of taxable income at roughly $170. The 1%/2% schedule actually gives about $226. It also left out the $60 renter's credit and leaned on vague 'disabled-related considerations' to reach zero, even though $225.78 minus $213 in credits leaves $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"By its own figures, about $233 of tax minus a $149 exemption credit leaves about $84. It still declared the liability 'effectively $0' with no basis, and it never applied the $60 renter's credit. Done correctly, $225.78 minus $213 leaves $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It understated the tax at about $200 instead of $225.78 and claimed a roughly $150 exemption credit fully offsets it. It also never applied the renter's credit explicitly. Both credits together total only $213, which leaves $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It used a projected 2026 standard deduction of about $5,880 and a $158 exemption credit. The applicable amounts are $5,706 and $153. With those figures it got $219 of tax and $1 of net liability instead of $225.78 minus $213, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"It added a second $144 exemption credit because the filer is disabled. California's extra exemption credit is for blindness only, not disability. It also used stale 2023 parameters ($5,540 deduction, $10,756 bracket) and left out the renter's credit; the correct credits total $213 against $225.78 of tax." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It computed $250–300 of tax, then claimed that a roughly $149 exemption credit plus nonexistent 'disability-related credits' brought it to zero. California has no disability nonrefundable credit, and the only credits here, the $153 exemption credit and the $60 renter's credit, total $213, which leaves $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It computed about $226 of tax and subtracted the $153 exemption credit. It left out the $60 nonrefundable renter's credit, which this renter qualifies for because AGI is below the limit, and so got about $73 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used AGI of $14,446, which leaves out the $8,000 taxable 403(b) distribution. Correct AGI is $22,534.34, which produces $225.78 of tax, more than the $213 in credits." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used AGI of $14,446 and a $5,363 deduction, which leaves out the $8,000 taxable 403(b) distribution. Correct AGI is $22,534.34, which yields $225.78 of tax that the $213 in credits do not fully offset." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used projected 2026 values: a $5,876 deduction, an $11,044 bracket and a $158 exemption credit. The applicable figures are $5,706, $11,079 and $153, and the misindexing gave $4.74 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It left out the $130 traditional IRA deduction, giving AGI of $22,665. It also used stale 2023 parameters, a $5,540 deduction and a $10,756 bracket, instead of $5,706 and $11,079, so it got $20.94 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used an approximate $5,800 standard deduction, a rounded $230 of tax and a $155 exemption credit. The correct figures are $5,706, $225.78 and $153, which give $12.78, not $15." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It claimed the single standard deduction exceeds AGI. AGI is $22,534.34 including the $8,000 403(b) distribution, well above the $5,706 deduction, so there is $225.78 of tax." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It said the exemption credit fully offsets the tax. That holds only if taxable income is understated, for example by dropping the $8,000 403(b) distribution. The correct $16,828.34 of taxable income produces $225.78 of tax, which exceeds the combined $213 of exemption and renter's credits." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its taxable income of $8,846 leaves out the $8,000 taxable 403(b) distribution. Correct taxable income is $16,828.34, and the resulting $225.78 of tax exceeds the $213 in credits." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave $0 with no derivation. That result fits only an understated income base, such as one that ignores the $8,000 403(b) distribution. The correct $225.78 of tax minus $213 in credits leaves $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,It reported roughly the pre-credit tax ($218) and did not subtract the nonrefundable credits this output nets out: the $153 personal exemption credit and the $60 renter's credit. Subtracting them leaves $12.78. +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It claimed the exemption credits fully eliminate the tax. On $16,828.34 of taxable income the 1%/2% schedule gives $225.78, and the credits total only $213, so $12.78 remains." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It set up the right structure with both brackets, the exemption credit and the $60 renter's credit. Its approximate parameters produced $17 instead of $225.78 minus $213, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It subtracted a $138 'personal exemption' from income as if it were a deduction and then said no nonrefundable credits apply. California's personal exemption is a $153 nonrefundable credit, and the $60 renter's credit also applies. It also used a stale $5,363 deduction and a $10,234 bracket." +us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It did not exclude the $2,778 traditional 401(k) deferral or deduct the $130 IRA contribution, giving AGI of $25,443 instead of $22,534.34. It also left out the $153 personal exemption credit and subtracted only the $60 renter's credit." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It asserted that the deduction and credits wipe out the tax without computing it. The 1%/2% schedule on $16,828.34 gives $225.78, which exceeds the $213 combined exemption and renter's credits by $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It applied American Opportunity Credit amounts against state tax, but California has no AOC equivalent. Its $72 matches $225.78 minus only the $153 exemption credit, which means it left out the $60 nonrefundable renter's credit." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"It added a $200 California 2.5% early-distribution additional tax on the 403(b) distribution. That tax is not part of this output, and the filer's disability exempts the distribution from it. It also wrongly treated the regular tax as fully offset, when $225.78 minus $213 leaves $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It claimed the exemption and renter's credits fully offset the tax. Those credits total $213 against $225.78 of tax on $16,828.34 of taxable income, so $12.78 remains." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used a 'projected' 2026 standard deduction and brackets instead of the published $5,706 deduction and $11,079 1% bracket that apply. That understated the tax and gave $4 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It assumed the nonrefundable credits fully offset the tax. The $153 exemption credit plus the $60 renter's credit total $213, which is less than the $225.78 of tax, leaving $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used an indexed 2026 standard deduction above $5,706 and larger credits. That cut net tax to $4 instead of $225.78 minus $213, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"Its structure was correct, but it rounded the result to $13. The exact computation is $225.78 of tax minus $153 and $60 in credits, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It used inflation-projected parameters instead of the $5,706 deduction, $11,079 bracket and $153 exemption credit. That understated net tax at $3 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used AGI of $14,446, which leaves out the $8,000 taxable 403(b) distribution. Correct AGI is $22,534.34, which yields $225.78 of tax, more than the $213 in credits." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed there was zero tax before credits. AGI of $22,534.34 minus the $5,706 deduction leaves $16,828.34 taxable, which gives $225.78 of tax." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used inflated brackets and a projected $158 exemption credit instead of $11,079 and $153. The result was $9 instead of $225.78 minus $213, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized a $7,499 medical deduction by counting pre-tax employer-sponsored insurance premiums and over-the-counter costs as deductible medical expenses. The allowable medical expenses do not clear 7.5% of AGI, so the $5,706 standard deduction applies, giving $225.78 of tax and $12.78 after credits." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used estimated 2026 parameters: a $5,843 deduction, an $11,345 bracket and a $157 exemption credit. The applicable figures are $5,706, $11,079 and $153, so it got $3 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,It subtracted only the $60 renter's credit and left out California's $153 nonrefundable personal exemption credit for a single filer. That left $169 instead of $12.78. +us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used AGI of about $14,446, which leaves out the $8,000 taxable 403(b) distribution. Correct taxable income is $16,828.34, and the resulting $225.78 of tax exceeds the $213 in credits." +us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for this output. The correct value is $225.78 of tax minus the $153 exemption and $60 renter's credits, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no value for this output. The correct value is $225.78 of tax minus the $153 exemption and $60 renter's credits, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It added the traditional 401(k) deferral and IRA deduction back into California AGI, but California conforms to both. It also left out the $153 personal exemption credit and the $60 renter's credit." +us,scenario_023,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used stale parameters: about $17,000 of taxable income from an older standard deduction, and a $149 exemption credit instead of $153. That gave $27 instead of $225.78 minus $213, which is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used the outdated $5,540 standard deduction and $149 exemption credit instead of $5,706 and $153. That overstated taxable income and understated the credit, giving $20 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"It claimed an extra $144 exemption credit for disability, but California's additional exemption credit applies only to blindness. It also left the 401(k) deferral and IRA deduction out of AGI. The correct credits are $153 plus $60, against $225.78 of tax." +us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It asserted zero tax after credits without computing it. On $16,828.34 of taxable income the tax is $225.78, which exceeds the $213 in exemption and renter's credits by $12.78." +us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model cut earned income to $14,665 by subtracting the 401(k) deferral, mixed in the 403(b) distributions, and arrived at $402. That is more than the childless CalEITC pays anywhere on its phase-out. The engine uses $17,442.65 of earnings, phases out on the higher $22,534 AGI, and gets $95.15." +us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model ran the phase-out on earned income of $17,443 alone. It skipped the rule that the credit is computed on AGI when AGI ($22,534) is higher than earnings. Because AGI sits further down the phase-out tail, the credit is $95.15, not $180." +us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model ruled out CalEITC because the filer has no qualifying dependent. CalEITC covers childless filers aged 18 or older, so this 28-year-old with $17,443 of wages qualifies for $95.15." +us,scenario_023,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated CalEITC as unavailable without qualifying children. Childless filers aged 18 or older are eligible, and this filer receives $95.15 after the AGI-based phase-out." +us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model first estimated a CalEITC of about $44, then raised it to $358 for unspecified 'credit interactions' that do not exist. The only refundable California credit here is CalEITC, which comes to $95.15 when phased out on the $22,534 AGI." +us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model answered $1,208, which is several times the maximum childless CalEITC. It applied a schedule meant for filers with children. The childless phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model assumed the childless CalEITC is fully phased out below $17,443 of earnings. CalEITC's phase-out tail runs past $30,000, so at AGI $22,534 the filer still gets $95.15." +us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,The model correctly phased out on AGI of about $22.5k but estimated the tail amount at $115. The credit on the CalEITC phase-out schedule at that AGI is $95.15. +us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model counted the $8,000 of taxable 403(b) distributions as investment income above the disqualified-income limit. Pension and retirement distributions are not disqualified investment income, which covers interest, dividends, rents, royalties, and capital gains. They only raise AGI, which reduces the credit to $95.15 rather than eliminating it." +us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model gave a near-maximum credit of $285 based on wages alone. It did not apply the phase-out using the higher AGI of $22,534, which brings the credit down to $95.15." +us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model placed only the $17,443 of earned income on the phase-out slope. Because AGI ($22,534) is higher than earnings, the credit is computed on AGI and comes to $95.15, not $150." +us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model misstated the inputs as $6,576 of earned income and $14,446 of AGI, then treated the credit as near its maximum of $300. The actual figures are $17,442.65 of earnings and $22,534 of AGI, deep in the phase-out tail, and they give $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model computed CalEITC as 85% of a federal EITC of $503. CalEITC is not a percentage of the federal credit; it has its own schedule. The childless federal EITC is also zero at AGI $22,534, and the standalone California schedule yields $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model said no CalEITC applies. A childless 28-year-old with $17,443 of earnings and $22,534 of AGI is within CalEITC's income limit and receives $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model invented a linear phase-out from a $300 maximum, running from $15,000 to $33,000, and used an AGI of $22,665. On the actual CalEITC tail schedule at AGI $22,534, the credit is $95.15, not $172.25." +us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model correctly phased out on the higher AGI of $22,535 but rounded its estimate to $100. The exact CalEITC schedule gives $95.15." +us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assumed this income level excludes the filer from all California refundable credits. The childless CalEITC extends past $30,000 of income, so it pays $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model based the credit mainly on W-2 earnings and applied only part of the AGI phase-out, arriving at $148. Computing fully on the higher AGI of $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model's $180 matches a credit read off the schedule at $17,443 of earnings. The phase-out must use the higher AGI of $22,534, which gives $95.15." +us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,The model gave $0 with no reasoning and missed that the childless CalEITC is refundable and available at this income. After the AGI-based phase-out it equals $95.15. +us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model said the filer's income is above the CalEITC limit. The limit is above $30,000, so AGI of $22,534 is still in the phase-out tail and yields $95.15." +us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model's $195 overstates the credit and matches a computation on wages without the AGI-based phase-out. Because AGI ($22,534) is higher than earnings, the credit is $95.15." +us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"The model correctly treated the filer as a childless CalEITC recipient in the phase-out range, but its $92 estimate falls short of the exact schedule. At AGI $22,534 the schedule gives $95.15." +us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model believed the childless CalEITC ends before AGI reaches $22,535. It confused CalEITC with the federal childless EITC's lower cutoff. CalEITC phases out above $30,000 and pays $95.15 here." +us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,The model cited a childless age test as the reason for a $0 credit. The 28-year-old filer meets CalEITC's minimum age of 18 and receives $95.15. +us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable California credit and missed the childless CalEITC. With $17,443 of earnings and $22,534 of AGI, that credit equals $95.15." +us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said no California refundable credits apply and overlooked CalEITC, which childless working filers aged 18 or older receive. Here it equals $95.15." +us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly placed the childless CalEITC in its 2026 phase-out range but estimated $114. The schedule evaluated at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model based the credit on $17,443 of earned income and did not switch to the higher AGI of $22,534, which governs the phase-out. That AGI reduces the credit to $95.15." +us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model evaluated the childless CalEITC at $17,443 of earned income only. Because AGI ($22,534) is higher than earnings, the credit is computed on AGI and equals $95.15, not $160." +us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model generated no California refundable credit. A childless filer aged 18 or older with $22,534 of AGI qualifies for a CalEITC of $95.15." +us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"The model correctly treated the childless CalEITC as phased out but applied too much reduction, landing at $25. The tail schedule at AGI $22,534 leaves $95.15." +us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model correctly used the higher AGI for the phase-out but estimated $106. The exact CalEITC schedule at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"The model concluded a childless filer at this income gets no CalEITC. CalEITC's income limit is above $30,000, so the credit at AGI $22,534 is $95.15." +us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"The model's childless phase-out schedule cut the credit to $28, which is too much reduction. The CalEITC tail at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model found no qualifying refundable credit and missed the childless CalEITC. For this filer it equals $95.15. +us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model made CalEITC depend on federal EITC eligibility. CalEITC is a separate credit, and its childless income limit is far above the federal phase-out point, so the filer receives $95.15 even with a federal EITC of zero." +us,scenario_023,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"The model reasoned that a $0 federal EITC means a $0 CalEITC. CalEITC is computed on its own schedule, which extends past $30,000, and pays $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model correctly used AGI of $22,535 because it is higher than earned income, but it overstated the tail amount at $120. The CalEITC schedule gives $95.15." +us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model treated the filer as getting roughly the maximum childless CalEITC of $300 and applied no phase-out. With AGI of $22,534, the credit is reduced to $95.15." +us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there is nothing to grade against the $95.15 CalEITC." +us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there is nothing to grade against the $95.15 CalEITC." +us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model claimed CalEITC is nonrefundable for childless filers. CalEITC is fully refundable for all eligible filers, and this childless filer receives $95.15." +us,scenario_023,state_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"The model correctly placed the childless CalEITC in its phase-out range but rounded to $100. The exact schedule at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model thought $14,665 of earnings was far above the childless CalEITC ceiling, confusing it with the federal childless EITC limit. CalEITC phases out above $30,000 and pays $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model added a $60 California renter's credit, which is nonrefundable, and inflated CalEITC to $251. The only refundable credit is CalEITC, which is $95.15 after the AGI-based phase-out." +us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model concluded that no California refundable credits apply and missed the childless CalEITC. For this 28-year-old worker with $22,534 of AGI it equals $95.15." us,scenario_023,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning used a $32,300 standard deduction ($100 too high) and reached about $7,019, close to the correct $7,031. It then submitted $3,325, a number that none of its own steps support. The final value does not match its own bracket calculation." @@ -1998,155 +2029,158 @@ us,scenario_025,payroll_tax,glm-5.3,llm_error,other,False,"The model's reasoning us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model said only the spouse's $62,725 of wages is taxed at 6.2% + 1.45%, which yields $4,798.46. Its $9,571.50 is about double that. This fits applying the combined 15.3% employer-plus-employee FICA rate instead of the 7.65% employee share." us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed payroll tax on 'annual wages for both spouses' and reported $9,106. The head has no listed wages, and the spouse's pension is not FICA wages. The correct base is only the spouse's $62,725, giving $4,798.48 at 7.65%." us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no parseable payroll_tax value and no reasoning. The correct derivation is 7.65% of the spouse's $62,725 wages, or $4,798.48." -us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Used $2,150 exemptions instead of the $1,900 that applies above $80,000 MAGI. Computed tax as 2.75% of the excess over $26,050 without the 2026 schedule's roughly $332 base amount, and skipped the $180.60 Ohio unreimbursed medical deduction. It then cut its own ~$1,576 figure to $875 with an unexplained 'graduated structure' adjustment." -us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the $1,900 exemptions and the $200 retirement income credit right. However, it taxed only 2.75% of income over $26,050 and left out the roughly $332 base amount in the 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction ($7,300 less 7.5% of FAGI), so it came out $327 short." -us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started from federal taxable income after subtracting a ~$28,000 federal standard deduction, but Ohio starts from federal AGI and has no standard deduction. It then applied the old 0.501%–5.75% bracket schedule instead of the 2026 schedule (about $332 plus 2.75% over $26,050). It also left out the $1,900 exemptions, the medical deduction, and the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Gave a rounded $1,000 with no computation. Applying the 2026 schedule (about $332 plus 2.75% over $26,050) to $90,944.70 of taxable income gives $2,116.61, and the $200 retirement credit brings it to $1,916.61. The $1,000 answer drops roughly half of that tax." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used $1,850 exemptions instead of $1,900 and skipped the $180.60 unreimbursed medical deduction. It padded its $1,792 figure with an arbitrary 'modest amount' rather than the schedule's roughly $332 base. It also said no nonrefundable credits apply, missing the $200 retirement income credit for the $32,200 pension." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Computed $1,790 from 2.75% over $26,050 without the 2026 base amount of about $332. It said the retirement and joint filing credits reduce the tax, then reported $2,426, which is higher than its own pre-credit figure. The joint filing credit does not apply because the head has no qualifying income, and the $180.60 medical deduction was skipped." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used $2,500 exemptions instead of the $1,900 for MAGI above $80,000, skipped the $180.60 unreimbursed medical deduction, and used the 2024 $360 base instead of about $332. Its stated parts ($360 + 2.75% × $63,875 = $2,116.56) don't add up to the $1,866 it reported. It never subtracted the $200 retirement income credit explicitly." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Explicitly assumed the 2026 schedule has no base amount. In fact tax above $26,050 is about $332 plus 2.75% of the excess. It also skipped the $180.60 unreimbursed medical deduction, which lowers Ohio AGI to $94,744.70, so it came out $327 short." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Stated that Ohio has no standalone deduction for health premiums. Ohio's unreimbursed medical care deduction (R.C. 5747.01(A)(10)) allows the $6,500 premiums plus $800 of other expenses above 7.5% of FAGI, which is $180.60 here. It also used $2,400 exemptions instead of $1,900 above $80,000 MAGI and left out the roughly $332 schedule base amount." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"Used $2,350 exemptions instead of $1,900 and left out the roughly $332 base amount. It then subtracted about $500 of 'retirement income and senior credits'. The retirement income credit is only $200, and the senior citizen credit requires age 65, which neither spouse has reached (61 and 57)." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Correctly used $3,800 of exemptions and the $200 retirement credit. However, it applied 2.75% only to the excess over $26,050 and left out the roughly $332 base amount in the 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used $1,000 exemptions instead of $1,900 and made up a 2.5%/3.5% bracket schedule instead of the 2026 schedule (about $332 plus 2.75% over $26,050). It left out the $200 retirement income credit for the $32,200 pension and the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied the pre-2024 multi-bracket schedule (3.226%, 3.688%) instead of the 2026 schedule (about $332 plus 2.75% over $26,050). It allowed only one $2,400 exemption instead of two $1,900 exemptions. It also said no nonrefundable credits apply, missing the $200 retirement income credit, and skipped the medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used $2,800 exemptions instead of the $1,900 for MAGI above $80,000 and left out the roughly $332 base amount. It said no nonrefundable credits apply, missing the $200 retirement income credit, and skipped the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied an outdated 3.5% rate above $46,100 instead of the 2026 schedule (about $332 plus 2.75% over $26,050) and used $2,400 exemptions instead of $1,900. It applied the 5% joint filing credit even though the head has no qualifying income. It also missed the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 employer-sponsored insurance premiums from wages, cutting FAGI to $73,717. Federal AGI is actually $94,925.30, so this inflated the 7.5%-floor medical deduction to $1,821 instead of $180.60. It also counted the $50 of over-the-counter expenses, used $2,400 exemptions instead of $1,900, and left out the roughly $332 base amount." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave no derivation. The correct figure is $2,116.61: about $332 plus 2.75% of $90,944.70 over $26,050. Subtracting the $200 retirement credit gives $1,916.61. The $1,695 answer leaves out the schedule's base amount." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The $1,762 answer equals 2.75% × ($90,125 − $26,050). That means it used $2,400 exemptions instead of $1,900, left out the roughly $332 base amount, skipped the $180.60 medical deduction, and never subtracted the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used $2,700 exemptions instead of the $1,900 for MAGI above $80,000, applied 2.75% without the roughly $332 base amount, and skipped the $180.60 medical deduction. It also never subtracted the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no derivation. The correct figure is $1,916.61: Ohio taxable income of $90,944.70 is taxed at about $332 plus 2.75% over $26,050 ($2,116.61), less the $200 retirement credit. The $1,280.50 answer leaves out the base amount and understates the 2.75% portion." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Correctly used $3,800 of exemptions and the $200 retirement credit. However, it applied 2.75% only to income over $26,050 and left out the roughly $332 base amount. It also skipped the $180.60 unreimbursed medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $1,694.06 answer equals 2.75% × ($94,925 − $26,050) minus $200. That means it subtracted neither the two $1,900 personal exemptions nor the $180.60 medical deduction, and it left out the roughly $332 base amount." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Wrongly said Ohio personal exemptions are $0 above $80,000 MAGI; they are $1,900 each. It also skipped the $180.60 medical deduction and the roughly $332 base amount, and never applied the $200 retirement income credit. These errors partly cancel, which is why the answer lands near the reference." -us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Subtracted a $30,000 federal standard deduction, which Ohio does not allow, and used $2,400 exemptions instead of $1,900. It then applied 2.75% to all taxable income, ignoring the $26,050 zero bracket and the roughly $332 base amount. It also missed the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its reasoning uses an obsolete 2.85%–4.396% bracket schedule and a joint filing credit that doesn't apply here, and arrives at $3,245. It then reported $1,260.80, which contradicts that work. Neither number follows the 2026 schedule (about $332 plus 2.75% over $26,050), the $1,900 exemptions, the medical deduction, or the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Treated the household as low-income with no tax. Ohio taxable income is $90,944.70, far above the $26,050 zero bracket, which gives $2,116.61 of tax and $1,916.61 after the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Applied a 'standard deduction', but Ohio has none. It starts from federal AGI, subtracts the $180.60 medical deduction and two $1,900 exemptions, and taxes the result at about $332 plus 2.75% over $26,050, less the $200 retirement credit. The $1,340 answer understates that result." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used the 2024 base amount of $360.69 instead of the 2026 schedule's roughly $332 for income over $26,050. It also added the $50 of over-the-counter expenses to the medical deduction ($230.63 instead of $180.60). Together these overstate the tax by about $27." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used $2,400 exemptions instead of the $1,900 for MAGI above $80,000 and left out the roughly $332 base amount. It said no nonrefundable credits apply, missing the $200 retirement income credit, and skipped the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $2,150.25 tax on $91,125 uses the 2024 base of $360.69 instead of the 2026 base of about $332. It also left out the $180.60 unreimbursed medical deduction, which lowers Ohio AGI to $94,744.70." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Excluded the $32,200 private pension from Ohio income, but Ohio fully taxes private pension income. Ohio's only pension relief is the nonrefundable retirement income credit, which is $200 for $8,000 or more of retirement income, not $25. It also applied a joint filing credit even though the head has no qualifying income." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Put the $21,208 employer-sponsored insurance premiums into the unreimbursed medical deduction, which produced a $21,388.63 deduction. Only the $6,500 out-of-pocket premiums and the $800 of other expenses count, so the deduction is $180.60. It also used $2,150 exemptions instead of $1,900." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Applied 2.75% only to income above the zero bracket and left out the 2026 schedule's base amount of about $332. It also skipped the $180.60 unreimbursed medical deduction, landing at $1,590 instead of $1,916.61." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Correctly used two $1,900 exemptions and the $200 retirement credit. However, it applied 2.75% only to the excess over $26,050 and left out the roughly $332 base amount. It also skipped the $180.60 unreimbursed medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Deducted the full $6,500 of premiums with no floor. Because the head can get employer-paid coverage, those premiums go on line 3 of Ohio's unreimbursed medical care worksheet with the $800 of other expenses. Only the amount above 7.5% of FAGI is deductible, which is $180.60 here." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Said Ohio taxable income drops to zero, but $94,744.70 of Ohio AGI less $3,800 of exemptions leaves $90,944.70. That amount is taxed at about $332 plus 2.75% over $26,050, giving $1,916.61 after the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Used $4,800 of exemptions instead of $3,800 (two $1,900 exemptions above $80,000 MAGI) and left out the roughly $332 base amount. It said there were no nonrefundable credits, missing the $200 retirement income credit, and skipped the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"Applied a 5% joint filing credit even though the head has no qualifying income, so the credit is unavailable. It also left out the roughly $332 base amount and the $180.60 unreimbursed medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Got the $1,900 exemptions, the $200 retirement credit, and the joint filing credit's unavailability right. However, it applied 2.75% only above the $26,050 zero bracket and left out the roughly $332 base amount. It also skipped the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Used an outdated 3.5% rate above $46,100 instead of the 2026 schedule (about $332 plus 2.75% over $26,050) and $2,650 exemptions instead of $1,900. It also missed the $200 retirement income credit and the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Used $4,800 of exemptions instead of $3,800 and left out the roughly $332 base amount. It said no nonrefundable credits apply, missing the $200 retirement income credit for the $32,200 pension, and skipped the $180.60 medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"Used $2,500 exemptions instead of the $1,900 for MAGI above $80,000, applied 2.75% without the roughly $332 base amount, and skipped the $180.60 medical deduction. It also never subtracted the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly used $1,900 exemptions and the $200 retirement credit. However, it applied 2.75% only to the excess over $26,050 and left out the roughly $332 base amount in the 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction." -us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"Applied senior or aged adjustments to cut taxable income to near zero, but neither spouse is 65. Ohio taxable income is $90,944.70, which gives $1,916.61 of tax after the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Applied no personal exemptions, though two $1,900 exemptions apply above $80,000 MAGI, and skipped the $180.60 medical deduction. It also left out the roughly $332 base amount and missed the $200 retirement income credit. These errors partly cancel." -us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Gave no derivation. Correctly, $90,944.70 of Ohio taxable income (after the $180.60 medical deduction and $3,800 of exemptions) is taxed at about $332 plus 2.75% over $26,050, less the $200 retirement credit, which gives $1,916.61. The $2,651 answer implies a higher rate and no retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Made up a $9,300 standard deduction, which Ohio does not have. It taxed the first $26,050 at 2.75% (that bracket is 0%) and used 3.5% above it, and its own sum doesn't add up. It also claimed a senior citizen credit, which requires age 65, for the 61-year-old head." +us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It reached about $1,576 by using $2,150 exemptions, leaving out the $332 base amount in Ohio's 2.75% bracket, and skipping the $180.60 medical deduction. It then cut that figure to $875 for an unspecified 'graduated structure' adjustment. The correct schedule is $332 plus 2.75% of the excess of $90,944.70 over $26,050, less the $200 credit, which gives $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It taxed only 2.75% of the excess over $26,050 and left out the $332 base amount in Ohio's bracket. It also skipped the unreimbursed-medical deduction ($7,300 above 7.5% of FAGI = $180.60), so it understated tax by about $327." +us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It started from federal taxable income after a federal standard deduction and applied obsolete Ohio brackets that run up to 5.75%. Ohio actually starts from federal AGI, allows only $1,900-per-person exemptions and the medical deduction, and taxes $332 plus 2.75% above $26,050. It also left out the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rounded $1,000 guess with no calculation. The correct path is OAGI of $94,744.70 after the $180.60 medical deduction, less $3,800 of exemptions, then $332 plus 2.75% of the excess over $26,050, less the $200 retirement income credit, which is $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used $1,850 exemptions, skipped the medical deduction, and swapped the $332 bracket base for an unexplained 'modest amount.' It also said no nonrefundable credits apply, so it missed the $200 retirement income credit for the spouse's $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It computed $1,790 without the $332 bracket base or the $180.60 medical deduction. It then named the retirement credit and a joint filing credit, which requires income for both spouses and does not apply here. Even so, it reported $2,426, a figure higher than its own pre-credit tax that does not follow from its arithmetic." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used $2,500 exemptions instead of $1,900 and the outdated $360 bracket base instead of $332. It skipped the medical deduction and took off about $250 of credits instead of the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It explicitly assumed the 2026 schedule has no base amount, so it taxed only 2.75% of the excess over $26,050 and left out the $332 base. It also skipped the $180.60 unreimbursed-medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of the $1,900 over-$80k tier and left out the $332 bracket base. It also wrongly said Ohio has no health-premium deduction, but Ohio deducts unreimbursed medical care, including the $6,500 of premiums, above 7.5% of FAGI ($180.60 here)." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used $2,350 exemptions and left out the $332 bracket base. It then subtracted about $500 of invented credits, including a senior credit that requires age 65 (the head is 61), instead of the single $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It applied 2.75% only to the excess over $26,050 and left out the $332 base amount in Ohio's bracket. It also skipped the $180.60 unreimbursed-medical deduction, which produced about $1,590 instead of $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It made up $1,000 exemptions and a 2.5%/3.5% bracket structure with no zero bracket. Ohio's actual structure is $1,900 exemptions, 0% up to $26,050, and $332 plus 2.75% above that. It also left out the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied an obsolete multi-bracket schedule (3.226%/3.688%) instead of the single 2.75% bracket with a $332 base. It allowed one $2,400 exemption instead of two $1,900 exemptions and said no nonrefundable credits apply, so it missed the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used $2,800 exemptions instead of $1,900, left out the $332 bracket base, and skipped the medical deduction. It also said no nonrefundable credits apply, missing the $200 retirement income credit for the pension." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied a 3.5% rate above $46,100, but Ohio's single 2.75% bracket with a $332 base runs to $100,000. It used $2,400 exemptions and took a 5% joint filing credit that is unavailable because the head has no income. It also missed the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored premiums from the spouse's wages, but they are not part of the listed wages, so FAGI stays at $94,925. That shrank the 7.5% floor and inflated the medical deduction to $1,821 (it also counted the $50 of OTC costs). It also used $2,400 exemptions and left out the $332 bracket base." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave $1,695 with no calculation. The correct path is the $180.60 medical deduction, $3,800 of exemptions, $332 plus 2.75% above $26,050, and the $200 retirement credit, for $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied 2.75% only to the excess over $26,050 on about $90,125 of income, which reflects $2,400 exemptions. It left out the $332 bracket base, the medical deduction, and the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used $2,700 exemptions instead of the $1,900 over-$80k tier and left out the $332 bracket base. It also skipped the medical deduction and the $200 retirement income credit for the $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave $1,280.50 with no derivation. That is far below the correct $332 plus 2.75% of $64,894.70, less the $200 retirement credit, which is $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied 2.75% only to the excess over $26,050 and left out the $332 base amount in Ohio's bracket. It also skipped the $180.60 unreimbursed-medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $1,694.06 equals 2.75% × ($94,925 − $26,050) − $200. So it allowed no personal exemptions (the correct amount is $3,800), skipped the medical deduction, and left out the $332 bracket base." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It wrongly said Ohio exemptions are $0 above $80,000 of MAGI; they are $1,900 each in that tier. It also left out the $332 bracket base, the medical deduction, and the $200 retirement income credit for the spouse's pension." +us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted a $30,000 federal standard deduction, which Ohio does not allow, and used $2,400 exemptions. It then applied 2.75% to all taxable income, ignoring both the $26,050 zero bracket and the $332 base. It also missed the medical deduction and the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied an obsolete five-bracket schedule (2.85%–4.396%) and a made-up 20% joint filing credit that is unavailable because the head has no income. Its reasoning ended at $3,245, yet it submitted $1,260.80, which matches neither its own arithmetic nor the correct $332 plus 2.75%-above-$26,050 calculation." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It treated the household as low-income with zero liability. Its $94,925 FAGI is far above Ohio's $26,050 zero bracket, so it owes $332 plus 2.75% of $64,894.70, less the $200 credit, which is $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It cited a standard deduction that Ohio does not have and showed no arithmetic for $1,340. It left out the $332 bracket base and the actual Ohio deductions: $180.60 of medical and $3,800 of exemptions." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used the outdated $360.69 bracket base instead of $332. It also put the $50 of over-the-counter expenses into the medical worksheet, which raised the deduction to $230.63 instead of $180.60, so it overstated the tax by about $27." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900 and left out the $332 bracket base. It said no nonrefundable credits apply, so it missed the $200 retirement income credit for the $32,200 pension and also skipped the medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its pre-credit tax of $2,150.25 on $91,125 implies the outdated $360.69 bracket base instead of $332. It also skipped the $180.60 unreimbursed-medical deduction above 7.5% of FAGI." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It excluded the $32,200 private pension from Ohio income, but Ohio fully taxes private pensions and gives only the $200 retirement income credit. It also used $2,150 exemptions, applied a joint filing credit the household doesn't qualify for, and shrank the retirement credit to $25." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It put the $21,208 of employer-sponsored insurance premiums into the medical worksheet and inflated the deduction to $21,388.63. Only the $6,500 of premiums and $800 of other expenses count ($7,300, giving a $180.60 deduction). The overstated deduction also pushed it into the $2,150 exemption tier instead of $1,900." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied 2.75% only to the excess over the zero bracket and left out the $332 bracket base. It also skipped the $180.60 medical deduction, producing about $1,590 instead of $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It computed 2.75% of the excess of $91,125 over $26,050 without the $332 base amount in Ohio's bracket. It also skipped the $180.60 unreimbursed-medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It deducted the full $6,500 of premiums with no floor. Because the head can get an employer-sponsored plan, those premiums belong on the unreimbursed-medical worksheet with the $800 of other expenses, subject to the 7.5%-of-FAGI floor, for a $180.60 deduction." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It claimed deductions and credits reduce Ohio tax to zero, but the household has about $90,945 of taxable income, far above the $26,050 zero bracket. Its only nonrefundable credit is $200 against $2,116.61 of tax." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900 and left out the $332 bracket base. It said there are zero nonrefundable credits, so it missed the $200 retirement income credit, and it also skipped the medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"It subtracted a 5% joint filing credit, but that credit requires each spouse to have qualifying income and the head has none. It also left out the $332 bracket base and the $180.60 medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It taxed only 2.75% of the excess over $26,050 and left out the $332 base amount in Ohio's bracket. It also skipped the $180.60 unreimbursed-medical deduction for premiums and expenses above 7.5% of FAGI." +us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied an obsolete 3.5% rate above $46,100 instead of Ohio's single bracket of $332 plus 2.75% up to $100,000. It used $2,650 exemptions instead of $1,900 and missed the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used $4,800 of exemptions instead of $3,800 and left out the $332 bracket base. It said no nonrefundable credits apply, missing the $200 retirement income credit, and it also skipped the medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used $2,500 exemptions instead of the $1,900 over-$80k tier and left out the $332 bracket base. It also skipped both the $180.60 medical deduction and the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied 2.75% only to the excess over $26,050 and left out the $332 base amount in Ohio's bracket. It also skipped the $180.60 unreimbursed-medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It invented senior or age-based adjustments that wipe out the tax, but neither spouse is 65 (they are 61 and 57). The household's $94,925 of income produces $2,116.61 of tax before the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It taxed only 2.75% of the excess over $26,050 and left out the $332 bracket base. It also skipped the $180.60 unreimbursed-medical deduction." +us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It allowed no Ohio exemptions, but two $1,900 exemptions apply. It left out the $332 bracket base and the medical deduction, and it said no nonrefundable credits apply, missing the $200 retirement income credit for the $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It gave a $2,651 bracket estimate with no derivation. It left out the $180.60 medical deduction, the $3,800 of exemptions, the $332-plus-2.75% schedule, and the $200 retirement credit, which together give $1,916.61." +us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It invented a $9,300 standard deduction and taxed the first $26,050 at 2.75% and the rest at 3.5%. Ohio actually taxes the first $26,050 at 0% and the rest at $332 plus 2.75%. It also applied a senior credit to the 61-year-old head, but that credit requires age 65." us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It computed MAGI of $82,301, below the 216% FPL limit (211% plus the 5% disregard), but assigned that band to CHIP. In NC that band is Medicaid coverage for older children (the OLDER_CHILD category), so the child is Medicaid-eligible and therefore excluded from CHIP." us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It understated household income at $61,093; wages minus traditional 401(k) and IRA contributions give about $82,301. It also labeled the 211% FPL threshold as NC Health Choice/CHIP, but that program was folded into Medicaid. An 11-year-old under that limit is Medicaid-eligible (OLDER_CHILD), which rules out CHIP." us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It set NC Medicaid for an 11-year-old at 133% FPL and treated NC Health Choice (211% FPL) as a separate CHIP tier. NC now covers children aged 6-18 up to 211% FPL through Medicaid (OLDER_CHILD), so this child is Medicaid-eligible and not CHIP-eligible." us,scenario_026,child1_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed the child above a lower Medicaid limit and under a separate NC Health Choice CHIP limit of 211% FPL. It missed that NC's 211% coverage for school-age children is Medicaid (OLDER_CHILD), and Medicaid eligibility excludes the child from CHIP." us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child1_chip_eligible. The correct derivation finds the 11-year-old Medicaid-eligible under NC's OLDER_CHILD category, which makes CHIP eligibility 0." -us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Used the old 133%/138% FPL limit for children ages 6–18. Since NC Health Choice merged into Medicaid, NC covers these children up to 211% FPL (216% with the disregard). Its MAGI of about $82,300 is 213% of the 2026 FPL for five, which is under that limit." -us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the income ratio right at about 213% FPL but compared it to a stale 133% FPL limit for children 6–18. NC's older-child Medicaid limit is 211% FPL (216% with the 5-point disregard), so Child 1 is eligible." -us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used a 200% limit and an understated FPL of $27,465 for five people; the 2026 figure is about $38,640. It also compared gross wages instead of MAGI ($82,301) and treated ESI as disqualifying, though Medicaid has no uninsured requirement. MAGI is 213% FPL, within NC's 211%+5% older-child limit." -us,scenario_026,child1_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"Said income was 'well above' the child Medicaid limit, which is consistent with the old 133–138% FPL standard. NC's older-child Medicaid category runs to 211% FPL (216% with the disregard), and the household's MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a 138% FPL (~$51,957) limit for NC children 6–18. After the Health Choice merger, NC Medicaid covers this group up to 211% FPL plus the 5-point disregard. MAGI of $82,301 is 213% FPL and qualifies." -us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Applied a 138% FPL limit, and miscalculated it at about $45,000 for a family of five. NC's actual older-child Medicaid limit is 211% FPL (216% with the disregard), which covers MAGI at 213% FPL." -us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Had the right 211% FPL limit but overstated income at about 260% FPL. MAGI is $85,209 in wages, which already include overtime, minus the $2,778 traditional 401(k) and $130 IRA deductions, or $82,301. That is 213% FPL, within 211% plus the 5-point disregard." -us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Put income at 219% FPL, which matches dividing by the 2025 guideline ($37,650) instead of the uprated 2026 FPL. That figure gives 213%. It also left out the 5-point MAGI disregard that raises NC's 211% older-child limit to 216%." -us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Medicaid has no uninsured requirement; that bar applies only to CHIP. On income, MAGI of 213% FPL is within NC's older-child Medicaid limit." -us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Added the $7,746 overtime premium on top of $85,209 in wages, even though wages already include overtime. It also left out the pre-tax 401(k) and IRA deductions and turned 211% FPL into about $67,000 using a wrong FPL. Correct MAGI is $82,301, or 213% of the 2026 FPL, which is within NC's 211%+5% older-child limit." -us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Used a 133% FPL limit for children 6–18. NC Medicaid now covers that age group up to 211% FPL (216% with the disregard), and the household's MAGI of about $82,300 is 213% FPL." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put household income at about $93k by adding the overtime premium to wages that already include overtime, and skipped the 401(k) and IRA deductions. Correct MAGI is $82,301, or 213% FPL, which is within NC's 211% older-child limit plus the 5-point disregard." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Calculated MAGI correctly at $82,301 but tested it against a 133% FPL limit. NC's Medicaid limit for children 6–18 is 211% FPL (216% with the disregard), so 213% FPL qualifies." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Calculated MAGI correctly at $82,301 but said it exceeded the child Medicaid limit. That $82,301 is 213% of the 2026 FPL for five, which falls within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard." -us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Compared gross wages ($85,209) to a 210% limit. It should have used MAGI after the 401(k) and IRA deductions ($82,301, or 213% FPL) and included the 5-point disregard, which brings NC's older-child Medicaid limit to 216%." -us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 147% FPL limit for children 6–18 in place of NC's 211% (216% with the disregard) older-child Medicaid limit. It also wrongly subtracted the head's ESI premiums from MAGI. Correct MAGI of $82,301 is 213% FPL and qualifies." -us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Said income was above the threshold without naming one, which is consistent with the old 133–138% FPL child limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Used a 133% FPL Medicaid limit and sent Child 1 to CHIP instead. NC merged Health Choice into Medicaid, so children 6–18 up to 211% FPL (216% with the disregard) are in Medicaid, and MAGI of $82,301 (213% FPL) qualifies." -us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used a 133% FPL limit for children. NC's older-child Medicaid limit is 211% FPL (216% with the disregard). It also wrongly took the head's ESI premiums out of MAGI, but even correct MAGI of 213% FPL qualifies." -us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Said Child 1 was not eligible without giving a reason, which is consistent with applying the old 133–138% FPL child limit. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income exceeded NC's limit without giving one, which is consistent with the old 133–138% FPL standard. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, which covers MAGI at 213% FPL." -us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Said income exceeded eligibility limits without a figure, which is consistent with the old 133–138% FPL child threshold. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), above the household's 213% FPL MAGI." -us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Said income exceeded NC's child Medicaid limits without naming one, which is consistent with the old 133–138% FPL threshold. After the Health Choice merger, NC covers children 6–18 up to 211% FPL plus the disregard, and MAGI of 213% FPL qualifies." -us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Medicaid has no uninsured requirement; only CHIP bars children with ESI. On income, MAGI of 213% FPL is within NC's older-child limit." -us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"Had the right 211% FPL limit but compared gross wages ($85,209) instead of MAGI ($82,301 after the 401(k) and IRA deductions). It also left out the 5-point disregard. MAGI of 213% FPL falls within the 216% effective limit." -us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Said earnings were above the child Medicaid threshold, which is consistent with the old 133–138% FPL limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Ruled out Medicaid from the wage level alone without testing the right threshold. MAGI of $82,301 is 213% of the 2026 FPL for five, which falls within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard." -us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said income exceeded the school-age child Medicaid limit, which is consistent with the old 133–138% FPL standard. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and the household is at 213% FPL." -us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Estimated NC's child Medicaid threshold too low; this answer is consistent with the old 133–138% FPL limit. The older-child limit is 211% FPL plus the 5-point disregard, which covers MAGI at 213% FPL." -us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's limit for an 11-year-old. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI of $82,301 is 213% of the 2026 FPL for five, so Child 1 is eligible." -us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Said income was above NC's child Medicaid limit, which is consistent with the old 133–138% FPL standard. The current older-child limit is 211% FPL plus the 5-point disregard, above the household's 213% FPL MAGI." -us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's Medicaid limit for age 11. NC's older-child Medicaid category runs to 211% FPL (216% with the disregard), which covers MAGI of $82,301 at 213% FPL." -us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"Said income exceeded the child Medicaid limit, which is consistent with the old 133–138% FPL threshold. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"Calculated MAGI correctly at $82,301 but said it exceeded NC's limit for five people. That MAGI is 213% of the 2026 FPL, within NC's older-child Medicaid limit of 211% FPL (216% with the disregard)." -us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"Said income exceeded NC's Medicaid threshold and cited ESI as an added bar. NC's older-child Medicaid limit is 211% FPL plus the disregard, which covers MAGI at 213% FPL. ESI does not disqualify a child from Medicaid; only CHIP has that bar." -us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"Used a 133% FPL limit for children 6–18. NC Medicaid covers this group up to 211% FPL (216% with the disregard), and MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Used a 133% FPL (~$51,000) limit for children 6–18 in place of NC's 211% (216% with the disregard) older-child Medicaid limit. It also understated MAGI at $74,555; the correct $82,301 is 213% FPL and qualifies." -us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"Used a 138% FPL limit for children 6–18. After the Health Choice merger, NC Medicaid covers this group up to 211% FPL plus the 5-point disregard, which includes MAGI at 213% FPL." -us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Used a 133% FPL limit and overstated income at about 240% FPL. Correct MAGI of $82,301 is 213% of the 2026 FPL, within NC's older-child Medicaid limit of 211% FPL plus the disregard." -us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Used a 133% FPL (~$50,000) limit for children 6–18. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), so the household's 213% FPL MAGI qualifies. The CHIP uninsured rule it cited does not apply to Medicaid." -us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for child1_medicaid_eligible, so nothing could be compared with the eligible result: MAGI at 213% FPL under NC's older-child limit." -us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Used a 133% (138% with disregard) FPL limit for children 6–18. NC's actual older-child Medicaid limit is 211% FPL (216% with disregard). Its income estimate of about 226% FPL also overstated MAGI, which is 213% of the 2026 FPL." -us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Said income was 'well above' NC's children's Medicaid limits, which is consistent with the old 133–138% FPL standard. NC Medicaid covers children 6–18 up to 211% FPL plus the disregard, and MAGI is 213% FPL." -us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,Put Child 1 in the 6–18 group but used a 133% FPL limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard). Even its own 198% FPL estimate would have qualified under that limit. -us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"Found income within the child Medicaid limit, then wrongly treated Child 1's employer-sponsored insurance as disqualifying. Medicaid has no uninsured requirement; only CHIP bars children with ESI." -us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Said income was too high for child Medicaid without a threshold, which is consistent with the old 133–138% FPL limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI of $82,301 is 213% FPL." +us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Used a 133% FPL limit (138% with the disregard) for NC children aged 6-18. Since NC Health Choice merged into Medicaid in 2023, the limit is 211% FPL (216% with the disregard). The model's ~$82,300 MAGI equals 2.13x FPL, which is under the real limit." +us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the ratio right at ~213% FPL but compared it with the outdated 133% FPL limit for ages 6-18. NC Medicaid now covers this age group up to 211% FPL (216% with the 5% disregard), so 213% qualifies." +us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used a family-of-5 FPL of ~$27,465 (far below the 2026 figure of ~$38,600), a 200% limit, and gross wages instead of MAGI. It also treated the child's employer coverage as disqualifying, but Medicaid has no uninsured requirement. Correct MAGI of $82,301 is 2.13x FPL, under NC's 216% effective limit for ages 6-18." +us,scenario_026,child1_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"Called the income 'well above' the child limit without naming the limit. MAGI of $82,301 (wages minus traditional 401(k) and IRA deductions) is 2.13x FPL, which is under NC's 211% FPL Medicaid limit for ages 6-18 (216% with the disregard). The 'No' answer matches using the pre-2023 133% limit." +us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a ~138% FPL limit (~$51,957) for ages 6-18, but NC Medicaid has covered this group up to 211% FPL (216% with the disregard) since NC Health Choice merged into Medicaid. MAGI at 2.13x FPL is under that limit." +us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Used a ~138% FPL limit and miscalculated it as ~$45,000 for a family of five. NC's actual limit for ages 6-18 is 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under it." +us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Put income at ~260% FPL, overstating MAGI: the overtime premium is already inside the $85,209 in wages, and the traditional 401(k) and IRA amounts must be subtracted. It also treated 211% as a hard cap without the 5-point disregard. Correct MAGI of $82,301 is 2.13x FPL, under the 216% effective limit." +us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Its ~219% FPL figure comes from dividing $82,301 by the 2025 family-of-5 FPL ($37,650) instead of the 2026 FPL used by the engine, which gives 2.13x. It also left out the 5-point MAGI disregard on top of NC's 211% limit for ages 6-18, so it marked an eligible child ineligible." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"Treated the child's employer-sponsored insurance as disqualifying. Medicaid has no uninsured requirement; that rule applies only to separate CHIP. On income, MAGI at 2.13x FPL is under NC's 211% (+5% disregard) limit for ages 6-18, so the child is eligible." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Added the $7,746 overtime premium on top of $85,209 in wages, although gross wages already include overtime. It also skipped the 401(k) and IRA deductions and understated 211% FPL as ~$67,000. Correct MAGI of $82,301 is 2.13x FPL, under NC's 216% effective limit for ages 6-18." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Compared ~$82k MAGI with a 133% FPL limit. NC Medicaid for ages 6-18 now runs to 211% FPL (216% with the disregard), so income at 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Overstated income at ~$93k by adding the overtime premium already counted in gross wages and skipping the 401(k)/IRA deductions. It also applied 210% with no 5-point disregard. Correct MAGI of $82,301 is 2.13x FPL, under NC's 216% effective limit for children aged 6-18." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but tested it against a 133% FPL ($51,205) limit. NC's Medicaid limit for ages 6-18 is 211% FPL (216% with the disregard), and $82,301 at 2.13x FPL is under it." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but said it exceeded NC's child limit. That figure is 2.13x the 2026 FPL, which is under NC's 211% FPL Medicaid limit for ages 6-18 once the 5-point disregard is applied (216%)." +us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Used gross wages of $85,209 instead of MAGI, which subtracts the $2,778 traditional 401(k) and $130 IRA deduction to reach $82,301. It also treated 210% FPL as a hard cap without the 5-point disregard. Correct MAGI is 2.13x FPL, under the 216% effective limit." +us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 147% FPL limit for ages 6-18, but NC's Medicaid limit after the Health Choice merger is 211% FPL (216% with the disregard). Even the model's own MAGI of $61,093 is under the correct limit, and the true MAGI of $82,301 (2.13x FPL) is too." +us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Said income was above the threshold without giving one. MAGI of $82,301 is 2.13x FPL, which is under NC's 211% FPL Medicaid limit for ages 6-18 (216% with the disregard). The answer matches using the pre-2023 133% limit." +us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied a 133% FPL Medicaid limit and sent older children above it to CHIP. NC folded Health Choice into Medicaid in 2023, so children aged 6-18 up to 211% FPL (216% with the disregard) are Medicaid-eligible. MAGI of $82,301 at 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied a 133% FPL limit to children aged 6-18 instead of NC's 211% FPL Medicaid limit (216% with the disregard). Both the model's $61,093 and the correct $82,301 MAGI (2.13x FPL) are under the real limit." +us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no derivation. MAGI of $82,301 is 2.13x FPL, under NC's 211% FPL Medicaid limit for the 6-18 group (216% with the disregard), so the child is eligible. The 'No' matches the outdated 133% limit." +us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income exceeded NC's limit without naming it. NC Medicaid covers children aged 6-18 up to 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Said income exceeded the limit without naming it. MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL Medicaid limit for ages 6-18 once the 5-point disregard is applied, so the child is eligible." +us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Said income exceeded NC's child limit, but NC Medicaid covers ages 6-18 up to 211% FPL (216% with the disregard). MAGI at 2.13x FPL is under it. The answer matches using the pre-merger 133% limit." +us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"Treated the child's employer-sponsored insurance as disqualifying, but Medicaid has no uninsured requirement; that rule applies only to separate CHIP. MAGI at 2.13x FPL is under NC's 216% effective limit for ages 6-18, so the child is eligible." +us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"Tested gross wages of $85,209 against a 211% cap. MAGI subtracts the $2,778 traditional 401(k) and $130 IRA deduction, giving $82,301 (2.13x FPL). A 5-point disregard then raises the effective limit to 216%, so the child qualifies." +us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Said earnings were above the threshold without naming it. MAGI of $82,301 is 2.13x FPL, under NC's 211% FPL Medicaid limit for school-age children (216% with the disregard). The 'No' matches using the outdated 133% limit." +us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Called the child ineligible without testing income against any limit. MAGI of $82,301 is 2.13x FPL, under NC's 211% FPL Medicaid limit for ages 6-18 (216% with the disregard), so the child qualifies with no special status needed." +us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said income was above the school-age limit for five people. NC Medicaid covers ages 6-18 up to 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which is under it." +us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Estimated a threshold below household income, but NC's Medicaid limit for ages 6-18 is 211% FPL (216% with the disregard). MAGI of $82,301 at 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's limit for an 11-year-old. MAGI of $82,301 is 2.13x FPL, under the 211% FPL Medicaid limit for ages 6-18 once the 5-point disregard brings it to 216%." +us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Said income was above NC's child limit. NC Medicaid covers ages 6-18 up to 211% FPL (216% with the disregard), and MAGI at 2.13x FPL is under it." +us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"Said income exceeded the age-specific limit, but the NC limit for ages 6-18 is 211% FPL (216% with the 5-point disregard). MAGI of $82,301 at 2.13x FPL is within it." +us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"Said income exceeded the child Medicaid limit without naming it. MAGI of $82,301 is 2.13x FPL, under NC's 216% effective limit for ages 6-18 (211% plus the 5-point disregard)." +us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but said it exceeded NC's limit for five people. That MAGI is 2.13x the 2026 FPL, under the 211% FPL limit for ages 6-18 once the 5-point disregard raises it to 216%." +us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"Said income was above NC's threshold and cited employer coverage as a second bar. MAGI at 2.13x FPL is under NC's 211% (+5% disregard) limit for ages 6-18, and employer coverage does not disqualify anyone from Medicaid." +us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"Applied a 133% FPL limit to children aged 6-18. Since NC Health Choice merged into Medicaid, the limit is 211% FPL (216% with the disregard), so MAGI at 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Applied a 133% FPL (~$51,000) limit and understated MAGI at $74,555 (the correct figure is $82,301). NC's Medicaid limit for ages 6-18 is 211% FPL (216% with the disregard), and 2.13x FPL is under it." +us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"Applied a ~138% FPL limit for ages 6-18. NC Medicaid covers that group up to 211% FPL (216% with the 5-point disregard), and MAGI at 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Overstated income at ~240% FPL (the correct figure is 2.13x) and applied the outdated 133% limit. NC's Medicaid limit for ages 6-18 is 211% FPL (216% with the disregard), so the child is eligible." +us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Limited Medicaid for ages 6-18 to 133% FPL and pointed to CHIP's uninsured rule. NC moved the 133-211% band into Medicaid itself, where coverage runs to 216% with the disregard and there is no uninsured requirement. MAGI at 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for child1_medicaid_eligible, so no answer could be parsed." +us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Applied a 133% FPL limit (138% with the disregard) and overstated income at ~226% FPL. NC's Medicaid limit for ages 6-18 is 211% FPL plus the 5-point disregard, and correct MAGI of $82,301 is 2.13x FPL, so the child is eligible." +us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Said income was 'well above' the child limit, but NC Medicaid covers ages 6-18 up to 211% FPL (216% with the disregard). MAGI of $82,301 at 2.13x FPL is under it." +us,scenario_026,child1_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"Said MAGI was above NC's child limit. MAGI of $82,301 is 2.13x FPL, under the 211% FPL Medicaid limit for ages 6-18 once the 5-point disregard is applied." +us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"Placed ages 6-18 in a 133% FPL Medicaid group. NC's limit is 211% FPL (216% with the disregard), so even the model's own ~198% estimate, and the correct 2.13x, both qualify." +us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"Found income within the child limit, then denied eligibility because the child has employer-sponsored insurance. Medicaid has no uninsured requirement; only separate CHIP excludes insured children. Income-eligible children aged 6-18 are Medicaid-eligible regardless of employer coverage." +us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Called income too high without naming a limit. MAGI of $82,301 is 2.13x FPL, under NC's 211% FPL Medicaid limit for ages 6-18 (216% with the disregard), so the child is eligible." us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It compared MAGI of $82,301 to a 216% FPL limit and labeled that the NC CHIP limit. That band is the NC Medicaid limit for children aged 6-18. The child qualifies for Medicaid under the older-child category, and Medicaid eligibility excludes CHIP, so the answer is 0, not 1." us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It found income of $61,093 below 211% FPL and treated that as NC Health Choice (CHIP) eligibility. At that income an 11-year-old in NC is Medicaid-eligible under the older-child category, since NC Health Choice is now part of Medicaid. A Medicaid-eligible child is not CHIP-eligible." us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It used the federal minimum of 133% FPL as the Medicaid ceiling for school-age children and put the 133-211% band in CHIP. NC Medicaid's older-child category covers an 11-year-old across that band, so at MAGI of $74,555 the child is Medicaid-eligible. Medicaid eligibility excludes CHIP." us,scenario_026,child2_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed household MAGI in a CHIP band above Medicaid and at or below 211% FPL, assuming NC Medicaid for an 11-year-old stops at a lower limit. The engine's NC Medicaid older-child category covers this child at this income. Because the child is Medicaid-eligible, the child is not CHIP-eligible." us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child2_chip_eligible, so there is no substantive answer to grade. The correct answer is 0: the child is Medicaid-eligible under NC's older-child category, which excludes CHIP." -us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but divided by a stale 5-person FPL of about $37,600–$37,900 and came out at 219%, when the 2026 FPL of about $38,700 gives 213%. It also compared against 211% without adding the 5-point MAGI disregard, which raises NC's effective limit for ages 6–18 to 216%." -us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It correctly put income at about 213% FPL but used the outdated 133% Medicaid limit for ages 6–18. NC Health Choice merged into Medicaid in 2023, so Medicaid now covers these children up to 211% FPL (216% with the disregard), and 213% qualifies." -us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It wrongly treated the household as having two earners (the head has no wages) and counted the child's employer-sponsored insurance against Medicaid eligibility. Medicaid has no uninsured requirement, and household MAGI of 2.13 times FPL is within NC's 216% effective limit for children ages 6–18." -us,scenario_026,child2_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said income was 'well above' the child limit, but MAGI is 2.13 times the 2026 FPL for five. NC Medicaid covers ages 6–18 up to 211% FPL plus the 5-point disregard (216%), so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2024 FPL for five ($36,580) instead of the 2026 value (about $38,700) and compared against 211% without the 5-point disregard. It then switched to a 133% Medicaid limit for ages 6–18, which NC abandoned when Health Choice merged into Medicaid. Correctly computed, MAGI is 2.13 times FPL, under the 216% limit." -us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used gross wages of about $85,000 without subtracting the $2,778 traditional 401(k) and $130 IRA deductions, used the 2024 FPL of $36,580, and treated 138% as NC's Medicaid limit with CHIP above it. NC Medicaid covers children 6–18 up to 211% plus the 5-point disregard, and MAGI of about $82,300 is 2.13 times the 2026 FPL, so the child qualifies." -us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"It overstated income at about 260% FPL. MAGI of about $82,300 (wages minus the traditional 401(k) and IRA deductions) is 2.13 times the 2026 FPL for five, which is under NC's 216% effective Medicaid limit for children ages 6–18." -us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Its figure of 219% FPL comes from dividing by the 2025 FPL for five ($37,650) instead of the 2026 value. MAGI is actually 2.13 times FPL, under NC's 211% limit plus the 5-point disregard (216%) for children ages 6–18." -us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It made employer-sponsored insurance a disqualifier, but Medicaid has no uninsured requirement. That rule belongs to CHIP. Child 2 qualifies as an older child because household MAGI of 2.13 times FPL is within NC's 216% limit." -us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It named roughly the right 216% limit but converted it to about $67,000 for five, when 216% of the 2026 FPL (about $38,700) is about $83,500. It also added overtime on top of wages, although the $85,209 already includes overtime. MAGI of about $82,300 is below the correct dollar limit." -us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used 133% FPL as the child Medicaid limit, but NC Medicaid has covered children 6–18 up to 211% (216% with the disregard) since Health Choice merged into it in 2023. Its $82,000 of income is 2.13 times the 2026 FPL, so the child qualifies." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It compared income against a bare 210% limit without the 5-point MAGI disregard. NC's effective limit for ages 6–18 is 216% FPL, and household MAGI is 2.13 times FPL, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but tested it against 133% FPL. NC Medicaid now covers children 6–18 up to 211% FPL plus the 5-point disregard (about $83,500 for five in 2026), so $82,301 qualifies." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but wrongly concluded that this exceeds NC's child limit. That income is 2.13 times the 2026 FPL for five, under the 216% effective older-child limit (211% plus the 5-point disregard)." -us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It tested gross wages of $85,209 instead of MAGI net of the traditional 401(k) and IRA deductions (about $82,300), and it used 210% without the 5-point disregard. Correct MAGI is 2.13 times FPL, under the 216% limit." -us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used 133% FPL as the Medicaid limit for ages 6–18, but NC Medicaid covers these children up to 211% FPL plus the 5-point disregard after the Health Choice merger. It also wrongly subtracted insurance premiums to get $61,093, when MAGI is about $82,300 (2.13 times FPL), which is still within the limit." -us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,It asserted that income is above the threshold without applying NC's actual limit. Household MAGI of 2.13 times FPL is within NC Medicaid's 216% effective limit (211% plus the 5-point disregard) for children ages 6–18. -us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but applied the pre-2023 133% FPL Medicaid limit and sent the child to CHIP. Since NC Health Choice merged into Medicaid, older children are Medicaid-eligible up to 211% FPL plus the 5-point disregard, which covers 2.13 times FPL." -us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a 133% FPL limit for children, but NC Medicaid covers ages 6–18 up to 211% FPL plus the 5-point disregard. Its income figure was also wrong: $61,093 rather than about $82,300 MAGI (2.13 times FPL), which still qualifies." -us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. MAGI of about $82,300 is 2.13 times the 2026 FPL for five, within NC Medicaid's 216% effective limit for children ages 6–18, so the answer should be eligible. Its 'not eligible' is consistent with applying an outdated 133% limit." -us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's limit without naming one. MAGI of 2.13 times FPL is under NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard (216%), so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,It said income exceeds the limit without applying NC's actual threshold. Household MAGI of 2.13 times FPL falls within the 216% effective Medicaid limit for children ages 6–18. -us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's child Medicaid limits, but NC Medicaid covers children 6–18 up to 211% FPL plus the 5-point disregard. The household's MAGI is 2.13 times FPL, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It estimated income at about 213% FPL but used a 133% Medicaid limit with separate CHIP up to 210%, a structure that ended when Health Choice merged into Medicaid in 2023. It also left out the 5-point disregard, which makes the effective Medicaid limit 216%. It wrongly cited the child's employer-sponsored insurance as a reason against eligibility." -us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"It compared gross wages of $85,209 against 211% FPL. Correct MAGI subtracts the traditional 401(k) and IRA deductions (about $82,300, or 2.13 times FPL), and the limit gets a 5-point disregard to 216%, so the child qualifies." -us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,It said earnings exceed the Medicaid threshold without computing one. Household MAGI of 2.13 times FPL is within NC's 216% effective Medicaid limit for school-age children. -us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed income was disqualifying without comparing MAGI to NC's limit. MAGI of about $82,300 is 2.13 times FPL, under the 211%-plus-5-point limit for children ages 6–18, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It treated $85,209 as 'well above' NC's child limit. MAGI net of the traditional 401(k) and IRA deductions is about $82,300, or 2.13 times the 2026 FPL for five, which is within the 216% effective limit for ages 6–18." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its estimated NC threshold was too low. NC Medicaid covers children 6–18 up to 211% FPL plus the 5-point disregard, and household MAGI is 2.13 times FPL, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It claimed MAGI exceeds NC's limit for an 11-year-old. MAGI is 2.13 times the 2026 FPL, under the 216% effective older-child limit (211% plus the 5-point disregard)." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income was above NC's child limit, but NC Medicaid covers ages 6–18 up to 216% FPL after the disregard. Household MAGI is 2.13 times FPL, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It said income exceeds the age-specific limit, but NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard. Household MAGI of 2.13 times FPL is under it." -us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It asserted that income exceeds the child limit without applying NC's 216% effective threshold for ages 6–18. Household MAGI is 2.13 times FPL, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but misjudged the limit. That is 2.13 times the 2026 FPL for five, below NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard (about $83,500)." -us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It cited employer-sponsored insurance as a disqualifier, but Medicaid has no uninsured requirement. It also judged income above the limit when MAGI of 2.13 times FPL is within NC's 216% effective limit for children 6–18." -us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It applied a 133% FPL Medicaid limit for children 6–18, but NC Medicaid has covered these children up to 211% FPL (216% with the disregard) since Health Choice merged into it. MAGI of 2.13 times FPL qualifies." -us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It applied the outdated 133% FPL limit for children 6–18 instead of NC's post-merger Medicaid limit of 211% FPL plus the 5-point disregard. Its MAGI was also wrong at $74,555 instead of about $82,300 (2.13 times FPL), which still qualifies." -us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It claimed MAGI exceeds NC's limit for ages 6–18, but that limit is 211% FPL plus the 5-point disregard. Household MAGI of 2.13 times FPL is under it." -us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It overstated income at about 240% FPL and used a 133% limit. MAGI is 2.13 times the 2026 FPL, and NC Medicaid covers children 6–18 up to 216% after the disregard, so Child 2 qualifies." -us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It applied a 133% FPL Medicaid limit for ages 6–18, but NC Medicaid covers these children up to 211% FPL plus the 5-point disregard. Household MAGI is 2.13 times FPL, so Child 2 qualifies as an older child regardless of employer-sponsored insurance." -us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value or explanation for child2_medicaid_eligible, so no answer was recorded." -us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133% limit (138% with the disregard) for children 6–18, but NC Medicaid covers them up to 211% (216% with the disregard). It also overstated income at 226% FPL when MAGI is 2.13 times the 2026 FPL." -us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It said income is 'well above' NC's child Medicaid limit, but household MAGI of 2.13 times FPL is within the 216% effective limit (211% plus the 5-point disregard) for ages 6–18." -us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It placed the child in NC's 6–18 group but capped that group at 133% FPL. NC Medicaid covers these children up to 211% FPL plus the 5-point disregard, so even its own 198% estimate, and the actual 2.13 times FPL, qualify." -us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a 133% limit and an understated FPL of $33,700 for five, then compared gross wages to it. NC Medicaid covers children 6–18 up to 211% FPL plus the 5-point disregard against the 2026 FPL of about $38,700, and MAGI of about $82,300 is 2.13 times FPL, which qualifies." -us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called household income too high without applying NC's threshold. MAGI is 2.13 times FPL, within the 216% effective Medicaid limit for children ages 6–18." +us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model used the right 211% FPL older-child limit but divided MAGI by a stale ~$37,600 poverty guideline, getting 219% FPL instead of the 2026 ratio of 2.13. It also left out the 5-point MAGI disregard, which raises the effective limit to 216% FPL. At 2.13× FPL the child is Medicaid-eligible." +us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model got the income ratio right (~213% FPL) but applied a 133% FPL Medicaid limit for ages 6–18. Since NC Health Choice was merged into Medicaid, the older-child Medicaid limit is 211% FPL (216% with the disregard), so 2.13× FPL qualifies." +us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model counted two wage earners, but the head has no wages. It also treated the child's employer-sponsored insurance as a bar to Medicaid, and Medicaid has no such rule. Household MAGI is 2.13× FPL, which is within NC's older-child Medicaid limit of 211%/216% FPL." +us,scenario_026,child2_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model called income 'well above' the child Medicaid limit, which only follows from using the outdated 133% FPL ceiling. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and household MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model first considered 211–215% but switched to a 133% FPL Medicaid limit for ages 6–18, treating the higher band as CHIP. It also used the 2024 FPL ($36,580). NC's Medicaid now covers children 6–18 up to 211% FPL (216% with the disregard), and MAGI of 2.13× the 2026 FPL qualifies." +us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated 138% FPL as the Medicaid limit and 211% as CHIP, then compared unreduced gross wages (~$85,000) to 211% of an outdated FPL. The 401(k) and IRA deductions bring MAGI down to 2.13× the 2026 FPL, which is under NC's merged Medicaid limit of 211%/216% FPL for children 6–18." +us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model put income at ~260% FPL, far above the actual MAGI of 2.13× FPL (wages minus the 401(k) and IRA deductions, divided by the 2026 FPL for 5). At 2.13× FPL the child is under NC's 216% older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model used an outdated poverty guideline and got 219% FPL instead of 2.13× FPL. It also set the child Medicaid limit below that figure, ignoring that NC's older-child Medicaid limit is 211% plus the 5-point disregard (216% FPL)." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model ruled out Medicaid because the child has employer-sponsored insurance. Medicaid has no uninsured requirement; that kind of bar applies to separate CHIP programs. With MAGI at 2.13× FPL, under NC's 216% older-child limit, the child is eligible." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model had the right ~216% FPL limit but converted it to about $67,000 for a family of 5. The correct figure is roughly $83,000 against the 2026 guideline. Household MAGI is 2.13× FPL, which is under 216%, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model applied a 133% FPL child Medicaid limit, which predates NC's merger of Health Choice into Medicaid. The older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI of about $82k is 2.13× FPL, which qualifies." +us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a 210% FPL limit and left out the 5-point MAGI disregard, which makes NC's older-child Medicaid limit 216% FPL. Household MAGI is 2.13× FPL, below that limit." +us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model compared MAGI of $82,301 to a 133% FPL limit. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard) since Health Choice was merged in, and MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 was about right, but it judged that figure over the limit, which only works with a limit far below NC's. The older-child Medicaid limit is 216% FPL (211% plus the disregard), about $83,000 for 5 in 2026, and MAGI of 2.13× FPL is under it." +us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model compared unreduced gross wages ($85,209) to a 210% FPL limit and left out the 5-point disregard. After the 401(k) and IRA deductions, MAGI is 2.13× FPL, which is under NC's 216% older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for ages 6–18. NC's merged Medicaid limit for older children is 211%/216% FPL, and household MAGI is 2.13× FPL, so the child is eligible. Separately, it wrongly subtracted the head's insurance premiums when computing MAGI." +us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income is above the threshold without naming a limit, so it was not using NC's older-child Medicaid limit of 211% FPL (216% with the disregard). Household MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a 133% FPL Medicaid limit and put the child in CHIP instead. Since NC folded Health Choice into Medicaid, children 6–18 up to 211% FPL (216% with the disregard) are covered by Medicaid, and MAGI of $82,301 (2.13× FPL) qualifies." +us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for children. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and household MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no income test. MAGI is 2.13× FPL, which is under NC's older-child Medicaid limit of 216% FPL (211% plus the disregard). A 'not eligible' answer only follows from using the outdated 133% limit." +us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeds NC's limit without naming the limit. NC's Medicaid limit for children 6–18 is 211% FPL (216% with the disregard), and MAGI is 2.13× FPL, which falls within it." +us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied an older-child limit below the household's 2.13× FPL MAGI. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard) since Health Choice was merged in, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model said income exceeds NC's child Medicaid limits, which only follows from the pre-merger 133% limit. The older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI is 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model got the ratio right (~213% FPL) but split coverage into Medicaid at 133% and CHIP at 210%, which is the pre-merger structure. Since Health Choice was folded in, NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard). It also wrongly cited employer-sponsored insurance as a factor." +us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"The model used the 211% limit but compared it to unreduced gross wages and left out the 5-point disregard. After the 401(k) and IRA deductions, MAGI is 2.13× FPL, under the effective 216% limit." +us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model said earnings are above the threshold, which only follows from applying a limit below 2.13× FPL. NC's Medicaid limit for school-age children is 211% FPL (216% with the disregard), so the child is eligible." +us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model ruled out eligibility based on income without testing it against NC's older-child Medicaid limit of 211% FPL (216% with the disregard). Household MAGI is 2.13× FPL, which qualifies." +us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model judged ~$85,209 'well above' the child Medicaid limit. After the 401(k) and IRA deductions, MAGI is 2.13× the 2026 FPL for 5, which is under NC's 216% FPL limit for older children (211% plus the disregard)." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's estimated NC child threshold was below 2.13× FPL. Since the Health Choice merger, the actual older-child Medicaid limit is 211% FPL (216% with the disregard), which covers this household's MAGI." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model said MAGI exceeds NC's limit for an 11-year-old. NC's limit is 211% FPL (216% with the disregard), and MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied a child Medicaid limit below 2.13× FPL. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), so this household's MAGI qualifies." +us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model said income exceeds the limit for age 11. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and household MAGI is 2.13× FPL, which is under it." +us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model said income exceeds the child Medicaid limit. NC's limit for ages 6–18 is 211% FPL (216% with the disregard) since Health Choice was merged in, and MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 was about right, but it judged that figure over the limit. NC's older-child Medicaid limit is 216% FPL (211% plus the disregard), about $83,000 for 5 in 2026, and MAGI of 2.13× FPL is under it." +us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model treated income as over NC's limit and counted employer-sponsored insurance as disqualifying. MAGI is 2.13× FPL, which is under the 211%/216% older-child Medicaid limit, and having ESI does not bar Medicaid." +us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for children 6–18. NC's merged Medicaid limit for that age group is 211% FPL (216% with the disregard), and MAGI is 2.13× FPL, so the child is eligible." +us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model compared MAGI of $74,555 to a 133% FPL limit. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and household MAGI of 2.13× FPL qualifies." +us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model said MAGI is above NC's limit for ages 6–18, which only follows from the outdated 133% limit. Since the Health Choice merger the limit is 211% FPL (216% with the disregard), and MAGI is 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model overstated income as ~240% FPL when MAGI is 2.13× FPL, and it applied a 133% limit. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), so the child is eligible." +us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for ages 6–18 and treated employer-sponsored insurance as disqualifying. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and MAGI of 2.13× FPL qualifies whether or not the child has ESI." +us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for child2_medicaid_eligible, so there was no answer to score." +us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 133%/138% FPL limit for children 6–18 and overstated income as ~226% FPL. MAGI is 2.13× FPL, and NC's merged Medicaid limit for older children is 211% FPL (216% with the disregard), so the child is eligible." +us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model said income is 'well above' NC's limit, which only follows from the pre-merger 133% ceiling. NC's limit for children 6–18 is 211% FPL (216% with the disregard), and MAGI is 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"The model applied an NC child Medicaid limit below 2.13× FPL. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), so the household's MAGI qualifies." +us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for ages 6–18. NC's Medicaid limit for that group is 211% FPL (216% with the disregard), so even its own ~198% FPL estimate qualifies, and the actual MAGI is 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit to an understated FPL of $33,700 and compared it to gross wages. NC Medicaid covers children 6–18 up to 211% FPL (216% with the disregard), and MAGI of 2.13× the 2026 FPL qualifies." +us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model said income is too high without applying NC's older-child Medicaid limit of 211% FPL (216% with the disregard). Household MAGI is 2.13× FPL, which is under that limit." us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 133%-211% FPL CHIP band for NC children aged 6-18, which assumes Medicaid for older children stops at 133% FPL. At its own 174% FPL figure, Child 3 falls under North Carolina's OLDER_CHILD Medicaid category. Medicaid eligibility excludes CHIP, so the answer is 0." us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It checked only whether $82,301 MAGI was under a 216% FPL 'CHIP limit' and never tested Medicaid first. Child 3 is Medicaid-eligible under the NC OLDER_CHILD category, and that Medicaid eligibility bars CHIP no matter where income falls against the CHIP ceiling." us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It concluded CHIP eligibility because $61,093 is under 211% FPL, treating that ceiling as a CHIP limit rather than checking Medicaid. At that income Child 3 qualifies for NC Medicaid under the OLDER_CHILD category, and Medicaid-eligible children are excluded from CHIP." @@ -2154,52 +2188,53 @@ us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,F us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"It treated 133% FPL as the top of Medicaid for a 9-year-old and put its $74,555 MAGI in a 133%-211% CHIP band. North Carolina's OLDER_CHILD Medicaid category covers Child 3 at this income, and Medicaid eligibility excludes CHIP." us,scenario_026,child3_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"It asserted that household MAGI was 'above Medicaid' and at or below 211% FPL, which puts the child in a separate CHIP tier. In fact Child 3 meets North Carolina's OLDER_CHILD Medicaid category, so the child is Medicaid-eligible and therefore not CHIP-eligible." us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"It stated outright that Child 3 is 'not Medicaid-eligible', and it measured income against a 6-person FPL even though the household has 5 people. Child 3 is Medicaid-eligible under North Carolina's OLDER_CHILD category, which bars CHIP, so 216% NC Health Choice limit is irrelevant." -us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 138% FPL Medicaid limit for ages 6-18 and sent 138-211% FPL to a separate CHIP. Since NC Health Choice merged into Medicaid in 2023, that band is Medicaid. Even its own estimate of about 197% FPL falls inside NC's 211%/216% Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model correctly computed income at about 213% FPL but used the old 133% FPL limit for children 6-18. NC's current Medicaid limit for that age group is 211% FPL plus the 5% disregard (216%), so 213% qualifies." -us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model turned a 207% FPL limit into $56,232, which is the dollar figure for a much smaller household. For five people, the limit is about $83,000 (216% of the 2026 FPL). It also failed to subtract the $130 IRA deduction, and its MAGI of about $82,300 is below that limit." -us,scenario_026,child3_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said income was 'well above' the limit without giving a figure. MAGI is $82,301, or 2.13 times FPL, which is below NC's 211%/216% FPL Medicaid limit for older children. Its answer matches using the old 133% limit." -us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model named the right 211% FPL limit but put it at about $66,000 for a family of five, when 211-216% of the 2026 FPL is about $81,500-$83,400. It also compared gross wages of $85,209 instead of MAGI of $82,301, which is net of the 401(k) and IRA deductions." -us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated 133% FPL as the Medicaid limit and 211% as a separate CHIP limit, but NC Health Choice was merged into Medicaid. It then compared gross wages of $85,209, not MAGI of $82,301, to an unadjusted 211% figure, leaving out the 5% disregard." -us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model put income at about 260% FPL, but MAGI of $82,301 is 2.13 times the 2026 FPL for five people. That is within NC's 216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model's 219% FPL figure comes from dividing $82,301 by the 2025 FPL for five ($37,650) instead of the 2026 guideline, which gives 2.13. It also left out the 5% disregard on NC's 211% Medicaid limit for older children, so it treated an eligible income as too high." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model ruled Child 3 out because of employer-sponsored insurance, but Medicaid has no ESI exclusion; the uninsured requirement applies only to CHIP. It also tested gross wages of $85,209 against a 210% limit built on a low FPL ($36,580), instead of testing MAGI of $82,301 (2.13 times FPL) against 216%." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 138% FPL limit for ages 6-18, but NC Medicaid now covers older children up to 211%/216% FPL. It also counted the $7,746 overtime twice, reaching more than $92,000, even though the $85,209 gross wages already include overtime." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model's MAGI of about $82,000 is right, but it tested it against the old 133% FPL limit. NC's Medicaid limit for older children is 211% plus the 5% disregard, and $82,301 is 2.13 times FPL, so Child 3 qualifies." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a 210% FPL limit with no 5% MAGI disregard and implicitly used gross income. MAGI of $82,301 is 2.13 times FPL, which is under NC's effective 216% Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is right, but it compared it to the outdated 133% FPL limit ($51,205). Since the NC Health Choice merger, NC Medicaid covers children 6-18 up to 211% FPL plus the 5% disregard." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly found MAGI of $82,301 but said it exceeds the limit. That amount is 2.13 times the 2026 FPL for five, below NC's 211%/216% FPL Medicaid limit for older children, so the model used a lower, pre-merger limit." -us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model compared gross wages of $85,209 to a 210% limit. It should have used MAGI of $82,301, after the $2,778 401(k) and $130 IRA deductions, and added the 5% disregard. At 2.13 times FPL, the child is eligible." -us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model subtracted Head's $21,208 ESI premiums from wages to reach a MAGI of $61,093, but MAGI is $82,301. It then used the outdated 133% FPL limit instead of NC's 211%/216% Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income was above the limit without naming one. MAGI of $82,301 (2.13 times FPL) is within NC's 216% FPL Medicaid limit for children 6-18, so the answer matches using the old 133% limit." -us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is right, but it used a 133% FPL Medicaid limit and sent the child to CHIP. NC Health Choice was absorbed into Medicaid in 2023, so children 6-18 are Medicaid-eligible up to 211% FPL plus the 5% disregard." -us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly subtracted the $21,208 ESI premiums to get a $61,093 income figure, and then applied the outdated 133% FPL limit. The real MAGI of $82,301 (2.13 times FPL) is under NC's 216% Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. MAGI of $82,301 is 2.13 times FPL, within NC's 211%/216% FPL Medicaid limit for older children. A 'not eligible' answer matches using the old 133% limit." -us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeds NC's limit without naming it. MAGI of $82,301 is 2.13 times FPL, below the 216% FPL Medicaid limit for children 6-18 that has applied since the NC Health Choice merger." -us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model gave no threshold. MAGI of $82,301 (2.13 times FPL) is within NC's 211%-plus-5%-disregard Medicaid limit for older children, so the answer matches using the old 133% limit." -us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model said income exceeds NC's limits without a figure. MAGI is 2.13 times FPL, below NC's 216% FPL Medicaid limit for children 6-18, so the model used an outdated, lower limit." -us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model correctly put income at about 213% FPL but split coverage into 133% Medicaid and 210% CHIP. Since the 2023 merger, NC Medicaid itself covers older children up to 211% plus the 5% disregard (216%). It also wrongly treated ESI as a bar to Medicaid." -us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used the right 211% limit but compared gross wages of $85,209 instead of MAGI of $82,301, which is net of the 401(k) and IRA deductions. It also left out the 5% disregard. At 2.13 times FPL, the child is eligible." -us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model said earnings were above the threshold without naming one. MAGI of $82,301 (2.13 times FPL) is within NC's 216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model judged income too high without doing any calculation. MAGI is 2.13 times the 2026 FPL for five, below NC's 211%/216% FPL Medicaid limit for children 6-18." -us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model judged gross earnings of about $85,209 to be 'well above' the limit. MAGI after the 401(k) and IRA deductions is $82,301, or 2.13 times FPL, which is within NC's 216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's estimated NC threshold is too low. NC Medicaid covers children 6-18 up to 211% FPL plus the 5% disregard, and this household's MAGI is 2.13 times FPL." -us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model said MAGI exceeds NC's limit for a 9-year-old. MAGI of $82,301 is 2.13 times FPL, below the 216% FPL Medicaid limit for older children that has applied since NC Health Choice merged into Medicaid." -us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model said income exceeds the limit without naming it. MAGI is 2.13 times FPL, within NC's 211%/216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied too low a Medicaid limit for age 9. NC covers children 6-18 up to 211% FPL plus the 5% disregard, and MAGI of $82,301 is 2.13 times FPL." -us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model said income exceeds the child Medicaid limit. MAGI of 2.13 times FPL is below NC's 216% FPL Medicaid limit for older children, so the model used the old, lower limit." -us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is right, but it said this exceeds NC's limit. That amount is 2.13 times the 2026 FPL for five, under NC's 211%-plus-5%-disregard Medicaid limit for children 6-18." -us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model cited employer coverage as disqualifying, but Medicaid has no ESI exclusion. It also understated the income limit: MAGI of 2.13 times FPL is within NC's 216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used the outdated 133% FPL limit for ages 6-18. Since NC Health Choice merged into Medicaid, the limit is 211% plus the 5% disregard, which MAGI of 2.13 times FPL meets." -us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model used an incorrect MAGI of $74,555 (the actual figure is $82,301) and tested it against the outdated 133% FPL limit. NC's current Medicaid limit for children 6-18 is 211%/216% FPL, and the household is at 2.13 times FPL." -us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model said MAGI exceeds the limit for ages 6-18. MAGI is 2.13 times FPL, below NC's 216% FPL Medicaid limit for older children, so the model used the old 133% limit." -us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model overstated income at about 240% FPL (actual 2.13) and used the old 133% limit. NC's Medicaid limit for older children is now 211% FPL plus the 5% disregard. -us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model used the outdated 133% FPL limit for ages 6-18 and treated ESI as disqualifying, which it is not for Medicaid. MAGI of 2.13 times FPL is within NC's 211%/216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for child3_medicaid_eligible, so its answer could not be scored." -us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a 133%/138% FPL Medicaid limit for children 6-18 and overstated income at 226% FPL. MAGI is 2.13 times the 2026 FPL, under NC's 216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model said income is well above NC's children's Medicaid limit. MAGI of $82,301 is 2.13 times FPL, within the 211%-plus-5%-disregard limit for ages 6-18." -us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model placed ages 6-18 in a 133% FPL Medicaid group, but NC's limit has been 211% plus the 5% disregard since the Health Choice merger. Its own estimate of about 198% FPL is under that limit." -us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the outdated 133% FPL limit with a too-low FPL of $33,700 for five, and compared gross wages instead of MAGI. MAGI of $82,301 is 2.13 times the 2026 FPL, within NC's 216% FPL Medicaid limit for older children." -us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model said income is too high without naming a limit. MAGI is 2.13 times FPL, below NC's 211%/216% FPL Medicaid limit for children 6-18." +us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 138% FPL Medicaid limit for ages 6-18 and moved the 138-211% band to CHIP, but NC folded NC Health Choice children into Medicaid, so the older-child Medicaid limit is 211% FPL (216% with the disregard). Its MAGI of $82,301 is correct and equals 2.13x the 2026 FPL, which is inside that limit." +us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model's ~213% FPL income ratio is correct, but it tested it against a 133% limit for ages 6-18. North Carolina's older-child Medicaid limit is 211% FPL plus the 5% disregard (216%), so 213% qualifies." +us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model gave a 207% FPL limit but turned it into $56,232, which badly understates the dollar threshold. 211% of the ~$38,680 2026 five-person FPL, plus the 5% disregard, is about $83,500, above the household MAGI of ~$82,301 (2.13x FPL)." +us,scenario_026,child3_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said income is well above the child Medicaid limit without computing it. Household MAGI is about $82,301, or 2.13x FPL, which is under NC's 211% (216% with disregard) Medicaid limit for children ages 6-18." +us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model named the correct ~211% limit but computed it as ~$66,000, implying an FPL near $31,300 instead of ~$38,680 for five people. It also compared gross wages instead of MAGI. 211% (216% with disregard) of the correct FPL is about $81,600-$83,500, which covers MAGI of $82,301 (2.13x FPL)." +us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the old 133% limit for children ages 6-18 and treated 211% as a separate CHIP ceiling. NC Medicaid now covers older children up to 211% FPL (216% with disregard). It also compared gross wages of $85,209 instead of MAGI net of the 401(k) and IRA deductions ($82,301, 2.13x FPL), which falls within that limit." +us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model overstated the household's income at ~260% FPL. The correct MAGI is $82,301, or 2.13x the 2026 five-person FPL, which is under NC's older-child Medicaid limit of 211% plus the 5% disregard." +us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model computed 219% FPL, consistent with dividing by the 2025 FPL ($37,650) instead of the 2026 FPL (~$38,680), and then treated that as over the child limit. MAGI is 2.13x the 2026 FPL, within NC's 211% (216% with disregard) older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated the child's employer-sponsored insurance as disqualifying for Medicaid, but having ESI does not bar Medicaid eligibility. It also compared gross wages to 210% of an understated FPL ($36,580) instead of MAGI ($82,301) against the 2026 FPL with the 5% disregard. That correct test gives 2.13x FPL, eligible under NC's older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 138% FPL limit for ages 6-18, but NC's older-child Medicaid limit is 211% FPL (216% with disregard). It also double-counted the overtime premium, which is already inside the $85,209 gross wages, to reach $92,000+. The correct MAGI is $82,301 (2.13x FPL)." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model's ~$82k MAGI is correct, but it applied a 133% FPL limit. North Carolina's Medicaid limit for children 6-18 is 211% FPL plus the 5% disregard, and $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model treated 210% FPL as a hard cap and skipped the 5% MAGI disregard. NC's older-child limit is 211% plus 5% (216%), and household MAGI of $82,301 is 2.13x FPL, so the child is eligible." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is correct, but it tested against 133% FPL. NC's Medicaid limit for children 6-18 is 211% FPL (216% with disregard), and $82,301 is 2.13x the 2026 FPL, which qualifies." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but used a child Medicaid limit below 213% FPL. NC's older-child Medicaid category runs to 211% FPL plus the 5% disregard, which covers 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model compared gross wages of $85,209 to 210% FPL. It should have used MAGI, which subtracts the $2,778 traditional 401(k) and $130 IRA contributions to reach $82,301, and applied the 5% disregard. That gives 2.13x FPL, under NC's 216% effective older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for ages 6-18, but NC's older-child Medicaid limit is 211% FPL (216% with disregard). Even its own understated MAGI of $61,093, which wrongly subtracted the head's $21,208 ESI premiums, falls well under that limit; the true MAGI of $82,301 is 2.13x FPL and also qualifies." +us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income is above the threshold without computing it. MAGI of $82,301 is 2.13x the 2026 FPL, which is within NC's 211% (216% with disregard) Medicaid limit for children 6-18." +us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a 133% FPL limit for older children and sent the 133-211% band to CHIP. NC moved those children into Medicaid with a 211% FPL limit (216% with disregard), so MAGI of $82,301 (2.13x FPL) is Medicaid-eligible." +us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit; NC's older-child Medicaid limit is 211% FPL plus the 5% disregard. Its $61,093 MAGI (which wrongly subtracted the head's ESI premiums) and the correct $82,301 MAGI (2.13x FPL) both fall within that limit." +us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no income test. MAGI of $82,301 is 2.13x the 2026 FPL, under NC's 211% (216% with disregard) older-child Medicaid limit, so the child is eligible." +us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeds the NC limit without computing it. MAGI of $82,301 is 2.13x FPL, within NC's 211% FPL (216% with disregard) Medicaid limit for children 6-18." +us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's 'exceeds limits' answer is consistent with the outdated 133% older-child threshold. NC's limit is 211% plus the 5% disregard, and MAGI of $82,301 is 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's 'exceeds limits' answer is consistent with the outdated 133% older-child threshold. NC's limit is 211% plus the 5% disregard, and MAGI of $82,301 is 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model correctly found ~213% FPL but used 133% for Medicaid and a separate 210% CHIP limit, leaving out the 5% disregard and NC's 2023 move of Health Choice children into Medicaid. The older-child Medicaid limit is 216% effective, so 213% qualifies, and ESI does not bar Medicaid." +us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model named the correct ~211% limit but compared gross wages of $85,209 instead of MAGI net of the traditional 401(k) and IRA contributions ($82,301), and it left out the 5% disregard. MAGI is 2.13x FPL, under the 216% effective limit." +us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model asserted earnings exceed the Medicaid threshold without computing it. MAGI of $82,301 is 2.13x the 2026 FPL, within NC's 211% (216% with disregard) older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model judged the wage level too high without any income test. MAGI of $82,301 is 2.13x FPL, which is within NC's 211% FPL (216% with disregard) Medicaid limit for children 6-18." +us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model called ~$85k well above the NC child limit, which implies the old 133%/138% threshold. NC's older-child Medicaid limit is 211% FPL plus the 5% disregard, and MAGI of $82,301 is 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's estimated NC child threshold is too low. The older-child Medicaid limit is 211% FPL (216% with disregard), and household MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model said MAGI exceeds NC's limit for a 9-year-old. That limit is 211% FPL plus the 5% disregard, and MAGI of $82,301 is 2.13x the 2026 FPL, which falls within it." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model treated income as above NC's child Medicaid limit. MAGI of $82,301 is 2.13x FPL, within the 211% (216% with disregard) older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model's 'exceeds the limit for this age' answer reflects the outdated 133% threshold. NC Medicaid covers children 6-18 up to 211% FPL (216% with disregard), and MAGI is 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model treated income as above the child limit. MAGI of $82,301 equals 2.13x FPL, under NC's 211% plus 5% disregard Medicaid limit for older children." +us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 is correct, but it used an NC child limit below that amount. The older-child Medicaid limit is 211% FPL plus the 5% disregard, and $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model treated income as over the NC limit and counted ESI as disqualifying, but ESI does not bar Medicaid. MAGI of $82,301 is 2.13x FPL, within NC's 211% (216% with disregard) older-child limit." +us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used a 133% FPL limit for ages 6-18. NC's older-child Medicaid limit is 211% FPL plus the 5% disregard, and MAGI of 2.13x FPL qualifies." +us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for ages 6-18, but NC's limit is 211% FPL (216% with disregard). Its own $74,555 MAGI and the correct $82,301 (2.13x FPL) are both under that limit." +us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model claimed MAGI is above NC's limit for ages 6-18. That limit is 211% FPL plus the 5% disregard, and MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model overstated income at ~240% FPL and used a 133% limit. Correct MAGI is $82,301, or 2.13x the 2026 FPL, under NC's 211% (216% with disregard) older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit and cited employer coverage, but NC's older-child Medicaid limit is 211% FPL plus the 5% disregard, and ESI does not bar Medicaid. MAGI of 2.13x FPL qualifies." +us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for child3_medicaid_eligible, so there is nothing to score. The correct derivation gives MAGI of 2.13x FPL, under NC's older-child Medicaid limit, so the child is eligible." +us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used the 133%/138% limit for ages 6-18 and overstated income at ~226% FPL by using gross wages against the 2025 FPL. MAGI of $82,301 is 2.13x the 2026 FPL, within NC's 211% (216% with disregard) older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model asserted income is well above the NC children's limit. MAGI of $82,301 is 2.13x FPL, within NC's 211% FPL (216% with disregard) Medicaid limit for children 6-18." +us,scenario_026,child3_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"The model treated MAGI as above NC's child limit. MAGI of $82,301 is 2.13x the 2026 FPL, under the 211% plus 5% disregard older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model placed age 9 in a 133% FPL group, but NC's older-child Medicaid limit is 211% FPL (216% with disregard). Its own ~198% FPL estimate and the actual 2.13x FPL are both within that limit." +us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 133% limit with an understated five-person FPL of $33,700 and compared gross wages. With MAGI of $82,301 and the ~$38,680 2026 FPL, income is 2.13x FPL, under NC's 211% (216% with disregard) older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted income is too high without a threshold test. MAGI of $82,301 is 2.13x FPL, within NC's 211% FPL (216% with disregard) Medicaid limit for children 6-18." us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It capped the nonrefundable child tax credit at $1,500 for three children; the 2026 CTC is $2,200 per child ($6,600 total) and its nonrefundable portion is applied against the full pre-credit liability before any refundable ACTC is computed, so the $4,586.51 of tax is entirely absorbed. It also used a $29,500 MFJ standard deduction instead of $32,200 and never took the $7,746.26 deduction for the qualified FLSA overtime premium. Its own stated arithmetic ($6,313 less $1,500 = $4,813) does not equal the $5,813 it submitted." us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its derivation reached the correct conclusion — the nonrefundable CTC absorbs the entire tentative tax, 'Result near $0' — and it then submitted $4,107, a figure no step in its own work produces. It also omitted the $7,746.26 qualified-overtime deduction that brings taxable income down to $42,354.26 and tentative tax to $4,586.51." us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It invented a ~$605 ceiling on the nonrefundable CTC by treating the $1,700-per-child refundable ACTC as consumed first; the nonrefundable credit is applied against the whole liability first and the refundable portion is the residual, so $6,600 of credit for three children eliminates the entire tax. It also missed the $7,746.26 qualified-overtime deduction, which lowers taxable income from its stated $50,001 to $42,354.26." @@ -2222,46 +2257,47 @@ us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error, us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of the $85,209 gross wages that already include it, then applied pre-TCJA parameters — a $16,200 standard deduction, $26,000 of personal exemptions, a 15% bracket, and a $1,000-per-child CTC. For 2026 the premium is instead deducted, the standard deduction is $32,200 with no exemptions, and the $6,600 credit for three children offsets the full $4,586.51 tentative tax." us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive computation reached the grader. The correct derivation yields $42,354.26 of taxable income after the $32,200 standard deduction and $7,746.26 overtime deduction, $4,586.51 of tentative tax, and $0 after the $6,600 nonrefundable child tax credit." us,scenario_026,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a fabricated $3,700 MFJ standard deduction in place of $32,200 and a fabricated $3,700 total child tax credit in place of $6,600, and it never subtracted the $2,778 traditional 401(k), the $130 IRA deduction, or the $7,746.26 overtime deduction. Its explanation states the credit fully offsets the tax while it submitted $5,931.15, contradicting itself." -us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It estimated tax at about $5,500 and then said the full $6,600 CTC was used nonrefundably, even though $5,500 is less than $6,600. It also skipped the $7,746 qualified overtime deduction, which lowers tax to about $4,587. That leaves $2,013.49 of CTC as refundable ACTC." -us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It assigned an arbitrary $1,000 refundable amount per child and never figured tax liability. The refundable CTC is the $6,600 credit minus the roughly $4,586.51 used against tax (after the standard and overtime deductions), which is $2,013.49." -us,scenario_026,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It said the nonrefundable CTC wipes out any refundable portion. However, pre-credit tax is only about $4,587 after the $32,200 standard deduction and the $7,746 overtime deduction, which is less than the $6,600 CTC. The $2,013.49 difference is refundable." -us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000 per child instead of $2,200 and a $31,500 standard deduction. It also left out the $7,746 qualified overtime deduction. Its own $5,600 tax is still below its $6,000 credit, yet it called the credit fully absorbed. The correct figures are about $4,586.51 in tax against a $6,600 CTC, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used $2,000 per child instead of the OBBBA $2,200 and assumed tax was high enough to absorb the credit. It never subtracted the $7,746 overtime deduction. Pre-credit tax is about $4,587, so $2,013.49 of the $6,600 CTC is refundable." -us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but left out the $7,746 FLSA overtime premium deduction, which overstated tax at about $5,450. With that deduction, tax is about $4,586.51, so the refundable ACTC is $2,013.49, not $1,150." -us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per-child CTC, a guessed $30,000 standard deduction instead of $32,200, and 2025 brackets. It also skipped the $7,746 qualified overtime deduction, which gave it $5,815 in tax. The correct tax is about $4,586.51, leaving $2,013.49 of the $6,600 CTC refundable." -us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of gross wages, even though gross wages already include it. The overtime premium should instead be a below-the-line deduction. It also used $2,000 per child and wrongly subtracted Roth contributions. Correct taxable income is about $42,355, giving $4,586.51 in tax and $2,013.49 of refundable CTC from the $6,600 credit." -us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 pre-credit tax is $45.36 lower than the correct $4,586.51, which equals 12% of $378. That means it took the OBBBA car-loan interest deduction for the $378 auto loan interest. No qualifying new U.S.-assembled vehicle is listed, so that deduction doesn't apply and the refundable CTC is $2,013.49." -us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of the 2026 OBBBA $6,600 and said tax used all of it. Because it ignored the $7,746 overtime deduction, it missed that tax is only about $4,587. That leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It got AGI of $82,301 right but said the nonrefundable CTC fully offsets tax with nothing left over. After the $32,200 standard deduction and the $7,746 overtime deduction, tax is about $4,586.51. The $6,600 CTC exceeds that by $2,013.49, which is refundable." -us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child credit ($3,000) as if the TCJA had expired. OBBBA made the CTC permanent at $2,200 per child, or $6,600, which exceeds the roughly $4,587 tax by $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $5,516.12 tax correctly reflects the $32,200 standard deduction but leaves out the $7,746 qualified overtime deduction. It also used $2,000 per child instead of $2,200. The correct figures are $4,586.51 in tax against a $6,600 CTC, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It reverted to 2017 rules, with a $1,000-per-child CTC and the $3,000 refundability threshold, instead of applying OBBBA's permanent $2,200-per-child credit and the overtime deduction. The correct figures are $6,600 in CTC against about $4,586.51 in tax, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It gave no reasoning, and its $0 answer treats the CTC as fully absorbed by tax. Tax is about $4,586.51 after the $32,200 standard deduction and the $7,746 overtime deduction. The $6,600 CTC exceeds that, leaving $2,013.49 of refundable ACTC." -us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child under the expired TCJA sunset) instead of OBBBA's $2,200 per child. The $6,600 credit exceeds the roughly $4,586.51 tax, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from wages, which gave it an AGI of $61,093 and a spurious $930 EITC. AGI is actually $82,301, so EITC is $0. The only refundable credit is the $2,013.49 of CTC left after about $4,586.51 of tax." -us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation, and its $0 answer ignores the refundable CTC. The $6,600 CTC ($2,200 for each of three children) exceeds the roughly $4,586.51 tax (after the standard deduction and the $7,746 overtime deduction) by $2,013.49, which is refundable." -us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It gave no derivation, and its $0 answer treats the CTC as entirely nonrefundable or used up. Tax is only about $4,586.51, so $2,013.49 of the $6,600 CTC is refundable as ACTC." -us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It claimed the nonrefundable CTC covers tax with nothing left over. Pre-credit tax is about $4,586.51 after the $32,200 standard deduction and the $7,746 qualified overtime deduction, so the $6,600 CTC leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It said tax fully absorbs the CTC, but it never compared the two amounts. Tax after the standard and overtime deductions is about $4,586.51, below the $6,600 CTC, which leaves $2,013.49 of refundable ACTC." -us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of $6,600 and calculated $5,676.12 in tax without the $7,746 qualified overtime deduction. The correct tax of $4,586.51 leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It paid the full $1,700-per-child refundable cap ($5,100) without first applying the CTC against tax. The ACTC is limited to the credit left after tax, which is $6,600 − $4,586.51 = $2,013.49." -us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It ruled out EITC correctly but gave no refundable CTC at all. The $6,600 CTC exceeds the roughly $4,586.51 pre-credit tax by $2,013.49, and that amount is refundable within the $1,700-per-child cap." -us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed no refundable credits apply and never figured the ACTC. With three qualifying children and tax of about $4,586.51, $2,013.49 of the $6,600 CTC remains and is refundable." -us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied post-2025 sunset rules with a nonrefundable $1,000 credit per child. OBBBA permanently set the CTC at $2,200 per child with up to $1,700 refundable, which produces $2,013.49 of refundable ACTC here." -us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its result implies pre-credit tax of $4,541.24, which is $45.36 (12% of $378) below the correct $4,586.51. It deducted the $378 auto loan interest as OBBBA qualified vehicle-loan interest, which doesn't apply here. The refundable CTC is $2,013.49." -us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but estimated tax at about $5,532 without the $7,746 qualified overtime deduction. With that deduction, tax is about $4,586.51 and the refundable ACTC is $2,013.49." -us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 pre-credit tax is $45.36 too low because it deducted the $378 auto loan interest as qualified passenger-vehicle loan interest. The correct tax is $4,586.51, which leaves $2,013.49 of the $6,600 CTC refundable." -us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It said this income level doesn't qualify for refundable CTC. ACTC doesn't phase out at $82,301 AGI. The $6,600 CTC exceeds the roughly $4,586.51 tax, so $2,013.49 is refundable." -us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child under the TCJA sunset) instead of OBBBA's $2,200 per child. The $6,600 credit exceeds the roughly $4,586.51 tax by $2,013.49, which is refundable." -us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its $4,671 pre-credit tax matches using the 2025 $31,500 MFJ standard deduction instead of the 2026 $32,200, which overstates tax by about $84. The correct tax is $4,586.51, leaving $2,013.49 refundable rather than $1,929." -us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a $200 Saver's Credit before the CTC. AGI of $82,301 exceeds the 2026 MFJ Saver's Credit income limit ($80,500), so no Saver's Credit applies. The CTC offsets the full $4,586.51 tax, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child) instead of OBBBA's $6,600. It also added the overtime premium to wages ($90,047) instead of deducting it, which pushed tax to $5,937. The correct tax is about $4,586.51, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_refundable_credits, so its answer is missing." -us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but calculated $5,873 in tax without the $7,746 qualified overtime deduction. With that deduction, tax is $4,586.51 and the refundable ACTC is $2,013.49." -us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It said the additional CTC is $0 without comparing the credit with tax. The $6,600 CTC exceeds the roughly $4,586.51 pre-credit tax by $2,013.49, which is refundable." -us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its $5,586.60 pre-credit tax is about $1,000 too high because it left out the $7,746 FLSA overtime premium deduction. The correct tax is $4,586.51, so $2,013.49 of the $6,600 CTC is refundable." -us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used $2,000 per child ($6,000) instead of OBBBA's $2,200 per child. It also calculated $5,479 in tax without the $7,746 overtime deduction. The correct figures are $6,600 in credit and $4,586.51 in tax, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It gave an unsupported $4,300 ACTC without capping refundability at the credit left after offsetting tax. The refundable CTC is $6,600 − $4,586.51 = $2,013.49." +us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It never took the OBBBA qualified-overtime deduction for the $7,746 FLSA premium, so it put taxable income at about $50k instead of about $42,355. It then claimed a $5,500 tax 'exceeds' the $6,600 CTC, which is backwards. The correct pre-credit tax is about $4,586.51, which leaves $2,013.49 of CTC refundable as ACTC." +us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It guessed about $1,000 per child of refundable credit without computing pre-credit tax. ACTC is limited to the CTC left over after the nonrefundable offset. Here that is $6,600 minus about $4,586.51 of tax, or $2,013.49, not $3,000." +us,scenario_026,federal_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It asserted that the nonrefundable CTC absorbs the whole credit. After the $32,200 standard deduction and the $7,746 qualified-overtime deduction, pre-credit tax is only about $4,586.51. That is less than the $6,600 CTC, so $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000-per-child CTC and the $31,500 standard deduction, and it skipped the $7,746 qualified-overtime deduction. It also called a $5,600 tax 'fully offset' by a $6,000 credit, ignoring that the $400 left over is refundable. Correctly, a $6,600 CTC against about $4,586.51 of tax leaves $2,013.49 of ACTC." +us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assumed tax liability covers a $6,000 CTC without computing the tax. It also missed the OBBBA $2,200-per-child amount and the $7,746 qualified-overtime deduction. Pre-credit tax is only about $4,586.51, so $2,013.49 of the $6,600 CTC is refundable." +us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but put pre-credit tax at about $5,450 because it never deducted the $7,746 FLSA overtime premium under the OBBBA qualified-overtime deduction. With that deduction, tax is about $4,586.51 and the refundable remainder is $2,013.49, not $1,150." +us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000 CTC and a roughly $30,000 standard deduction instead of 2026's $32,200, and it skipped the $7,746 qualified-overtime deduction. That inflated tax to $5,815 and left only $185 of ACTC. The correct figures are about $4,586.51 of tax against a $6,600 CTC, leaving $2,013.49." +us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of wages, which already include overtime, and pushed AGI to about $89,000. The overtime premium is actually a below-the-line deduction under OBBBA. With the correct AGI of $82,301 and that deduction, tax is about $4,586.51, below the $6,600 CTC, so $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its pre-credit tax of $4,541.24 implies it deducted the $378 auto loan interest as qualified passenger-vehicle loan interest. Nothing in the facts makes the loan qualifying, since no new US-assembled vehicle is listed, so the deduction is $0. Without it, tax is about $4,586.51 and the ACTC is $2,013.49, not $2,058.76." +us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $6,000 CTC and assumed tax absorbs it fully, never applying the $7,746 qualified-overtime deduction or the $32,200 standard deduction. Pre-credit tax is only about $4,586.51, so $2,013.49 of the $6,600 CTC is refundable." +us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It got AGI right at $82,301 but never applied the OBBBA qualified-overtime deduction for the $7,746 FLSA premium, so it overstated tax and concluded the CTC was absorbed. Tax after the $32,200 standard deduction and the overtime deduction is about $4,586.51, leaving $2,013.49 of the $6,600 CTC refundable." +us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the TCJA-sunset $1,000-per-child CTC ($3,000 total) instead of the OBBBA $2,200-per-child credit ($6,600) in effect for 2026. It also ignored the qualified-overtime deduction. Against about $4,586.51 of tax, $2,013.49 of the $6,600 credit is refundable." +us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $5,516.12 tax is correct only before the OBBBA qualified-overtime deduction. Taking the $7,746 FLSA premium off taxable income cuts tax to about $4,586.51. It also used $2,000 per child instead of the 2026 $2,200, so the ACTC is $6,600 minus $4,586.51, or $2,013.49." +us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the CTC reverts to 2017 rules in 2026 ($1,000 per child, $3,000 refundability threshold), but OBBBA made the $2,200-per-child credit with a $1,700 refundable cap permanent. Its $2,965.85 tax also reflects pre-OBBBA brackets and deductions. The correct result is a $6,600 CTC against about $4,586.51 of tax, leaving $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave a bare assertion that no refundable credit applies and never computed tax liability against the CTC. After the $32,200 standard deduction and the $7,746 qualified-overtime deduction, tax is about $4,586.51. The $6,600 CTC exceeds that by $2,013.49, which is refundable ACTC." +us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used the TCJA-sunset $3,000 CTC instead of the OBBBA $2,200 × 3 = $6,600 credit, so it concluded the credit was fully absorbed. Pre-credit tax is about $4,586.51 once the qualified-overtime deduction is applied, leaving $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from AGI to get $61,093. Those premiums are not an above-the-line deduction, and that error produced a $930 EITC that does not exist. AGI is $82,301, EITC is $0, and the ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It reported zero refundable credits without comparing the $6,600 CTC to tax liability. After the $32,200 standard deduction and the $7,746 qualified-overtime deduction, tax is about $4,586.51, so $2,013.49 of CTC is refundable." +us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave zero with no computation, missing that tax liability is only about $4,586.51 once the $32,200 standard deduction and the $7,746 OBBBA overtime deduction are taken. The unused $2,013.49 of the $6,600 CTC is refundable ACTC." +us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It asserted that the nonrefundable CTC covers the credit, but it overstated tax by skipping the qualified-overtime deduction for the $7,746 FLSA premium. Tax is about $4,586.51 against a $6,600 CTC, leaving $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It claimed the nonrefundable CTC is fully absorbed. Pre-credit tax after the $32,200 standard deduction and the $7,746 qualified-overtime deduction is about $4,586.51, which is less than the $6,600 CTC, so $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) and a pre-credit tax of $5,676.12 that leaves out the OBBBA qualified-overtime deduction for the $7,746 premium. The correct figures are a $6,600 CTC against about $4,586.51 of tax, giving $2,013.49 of ACTC." +us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It treated the $1,700-per-child refundable cap as the refundable amount. ACTC is limited to the CTC left after it offsets tax liability. That remainder is $6,600 minus about $4,586.51, or $2,013.49, well below the $5,100 cap." +us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It cited irrelevant age-6 effects and never compared the CTC to tax liability. A $6,600 CTC against about $4,586.51 of tax (after the $32,200 standard deduction and the $7,746 overtime deduction) leaves $2,013.49 of refundable ACTC." +us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed income rules out every refundable credit. The refundable CTC has no low phase-out at this income. With pre-credit tax of about $4,586.51 against a $6,600 CTC, $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the TCJA-sunset $1,000 nonrefundable child credit, ignoring OBBBA's permanent $2,200-per-child CTC with a $1,700 refundable portion. The $6,600 CTC exceeds the roughly $4,586.51 tax by $2,013.49, which is refundable." +us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its pre-credit tax of $4,541.24 matches taxable income of $41,977, meaning it deducted the $378 auto loan interest as qualified vehicle loan interest. No qualifying new US-assembled vehicle loan is listed, so that deduction is $0. Tax is about $4,586.51 and the ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its pre-credit tax of about $5,532 matches taxable income before the OBBBA qualified-overtime deduction. Deducting the $7,746 FLSA premium cuts tax to about $4,586.51, so $2,013.49 of the $6,600 CTC is refundable, not $1,068." +us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 tax reflects deducting the $378 auto loan interest under the OBBBA car-loan-interest provision. The facts establish no qualifying new US-assembled vehicle, so the deduction is $0. Pre-credit tax is about $4,586.51 and the ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It claimed the household fails to qualify for refundable CTC 'at this income level', but nothing phases the CTC out at $82,301 AGI. The $6,600 credit exceeds the roughly $4,586.51 tax, and the $2,013.49 difference is refundable." +us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the TCJA-sunset $3,000 CTC instead of the OBBBA $6,600 credit for three children, and it missed the qualified-overtime deduction. Tax of about $4,586.51 against a $6,600 CTC leaves $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its $4,671 tax matches using the 2025 standard deduction of $31,500 instead of 2026's $32,200, with the overtime deduction and 2026 brackets otherwise correct. With $32,200, tax is about $4,586.51 and the ACTC is $2,013.49, not $1,929." +us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It claimed a $200 Saver's Credit even though AGI of $82,301 is above the 2026 MFJ Saver's Credit AGI ceiling. That shifted $200 of CTC from nonrefundable to refundable. Without it, the $6,600 CTC against about $4,586.51 of tax leaves $2,013.49 of ACTC." +us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It added the $7,746 overtime premium on top of wages that already include it, inflating earned income to $90,047, when the premium should have been a deduction. It also used the TCJA-sunset $3,000 CTC instead of $6,600. Correct tax of about $4,586.51 leaves $2,013.49 of ACTC." +us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_refundable_credits, so there was no answer to compare with the $2,013.49 ACTC." +us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but put pre-credit tax at $5,873, never subtracting the $7,746 FLSA overtime premium under the OBBBA qualified-overtime deduction. Correct tax is about $4,586.51, leaving $2,013.49 refundable, not $727." +us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It treated EITC ineligibility as ruling out the additional CTC as well. ACTC depends only on the CTC left after offsetting tax. A $6,600 CTC against about $4,586.51 of tax leaves $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,muse-wonder,llm_error,other,False,"It had the right structure ($6,600 CTC, overtime deduction applied, no EITC) but rounded pre-credit tax to about $4,581 instead of $4,586.51. That approximation produced $2,019 instead of $2,013.49." +us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,other,False,"Its pre-credit tax of $5,586.60 is exactly $1,000 above the correct figure of about $4,586.60 (the 12% bracket portion on $17,555 is $2,106.60, not $3,106.60). That arithmetic slip cut the ACTC to $1,013.40 instead of $2,013.49." +us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000-per-child CTC and a pre-credit tax of $5,479 that leaves out the $7,746 qualified-overtime deduction. The correct figures are a $6,600 CTC against about $4,586.51 of tax, giving $2,013.49 of ACTC." +us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It asserted $4,300 of ACTC with no link to tax liability. ACTC equals the CTC left after the nonrefundable offset, which is $6,600 minus about $4,586.51, or $2,013.49, within the $5,100 cap." us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the income comparison by calling $85,208.82 well below the free-meal standard. That income is 220% of the family-of-five poverty guideline, and without SNAP/TANF categorical eligibility or universal free meals in North Carolina, the household must pay full price." us,scenario_026,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -2280,42 +2316,43 @@ us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,t us,scenario_026,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the wrong tier of the NC child deduction: $500 per child instead of the $1,500 that applies to MFJ AGI of $80,001–$100,000. That gave a $1,500 total instead of $4,500 and taxable income of $55,301 instead of $52,301. It then submitted $2,882, which does not match its own figure of $2,206.51." -us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated federal taxable income as NC's starting point and subtracted a $29,200 federal standard deduction. The correct path starts from federal AGI and subtracts the $25,500 NC standard deduction plus the $4,500 NC child deduction. It also used a 4.75% rate instead of 3.99% and invented a dependent credit. Its final $5,238 has no connection to any of its own steps." -us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed NC deductions and exemptions wipe out this household's tax, but NC has no personal exemptions. The $25,500 standard deduction and $4,500 child deduction leave $52,300.52 taxable at 3.99%, so the tax is $2,086.79, not zero." -us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left the $130 traditional IRA deduction out of AGI ($82,431) and used $2,500 per child for the NC child deduction instead of the $1,500 tier for MFJ AGI of $80,001–$100,000. It then switched between 4.25% and 3.99% and submitted $3,041, a number none of its calculations produce." -us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the NC child deduction entirely ($1,500 × 3 = $4,500) and also skipped the $130 traditional IRA deduction. As a result it taxed $56,931 instead of $52,300.52 at 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used $21,500 as the NC MFJ standard deduction instead of $25,500. It also applied the top child-deduction tier of $3,000 per child, which only covers MFJ AGI up to $40,000; the correct tier at $82,301 is $1,500 per child. The two errors partly offset each other, giving $30,500 of deductions instead of $30,000." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 FLSA overtime premium to wages that already include it, which inflated AGI to $90,047. It also left out the NC child deduction and used 4.25% instead of the 2026 rate of 3.99%. It then raised the result to $3,350 for an unspecified 'addback'." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It placed AGI of $82,301 in the right $80,001–$100,000 bracket but assigned it $1,000 per child instead of $1,500. That made the NC child deduction $3,000 instead of $4,500, so it taxed $53,801 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $7,746 overtime premium twice, on top of gross wages that already include it, which gave AGI of $90,047. It also left out the $4,500 NC child deduction, so it taxed $64,547 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI and standard deduction were right, but it left out the NC child deduction of $1,500 for each of three children. That left $56,801 taxable instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It left out the $4,500 NC child deduction, so it taxed $56,801 instead of $52,300.52. It also used a 4.0% rate instead of the 2026 statutory rate of 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the $25,500 standard deduction from AGI. It skipped the NC child deduction of $1,500 per child ($4,500), which overstated taxable income by $4,500." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from the spouse's wages, which cut AGI to $61,093. The head has no wages, so those premiums do not reduce AGI. Starting from that low AGI, it chose the $2,000-per-child tier instead of the $1,500 tier that applies at the correct AGI of $82,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. At 3.99%, its $3,538 implies about $88,700 of taxable income, which means it skipped the $25,500 NC standard deduction and the $4,500 child deduction that bring taxable income down to $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It treated the head's $21,208 ESI premiums as a reduction to the spouse's wages, which produced AGI of $61,093. The head has no wages, so the premiums do not reduce AGI. The low AGI also pushed it into the $2,000-per-child tier instead of the correct $1,500 tier." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,538.77 implies taxable income near $88,700 at 3.99%, which means it skipped the $30,000 of NC standard and child deductions applied to $82,300.52 of AGI." -us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a 3.74% NC rate instead of the 2026 statutory rate of 3.99%. It also subtracted a $130 spousal IRA contribution for the head that the inputs do not list, and it left out the $4,500 NC child deduction." -us,scenario_026,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used $4,000 per child for the NC child deduction. The correct amount for MFJ AGI of $80,001–$100,000 is $1,500 per child, so it deducted $12,000 instead of $4,500 and understated taxable income at $44,801." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions eliminate NC taxable income, but the $25,500 standard deduction and $4,500 child deduction total only $30,000. That leaves $52,300.52 taxable at 3.99%, or $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no real derivation. At 3.99%, its $4,875 implies about $122,000 of taxable income, far more than the household's $82,300.52 AGI before any deductions. The figure ignores the $30,000 of NC deductions and the actual income level." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,957.29 equals 3.99% of $49,055. That reflects subtracting the $7,746 overtime premium to get AGI of $74,555 even though overtime does not reduce AGI, and leaving out the $4,500 NC child deduction." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $2,027 equals 3.99% of $50,801, which means it applied a $2,000-per-child deduction ($6,000). That is the tier for MFJ AGI of $60,001–$80,000; the correct amount at AGI of $82,300.52 is $1,500 per child ($4,500)." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left out the NC child deduction ($4,500) and the $130 traditional IRA deduction, so it taxed about $56,931 instead of $52,300.52 at 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used $1,000 per child ($3,000) for the NC child deduction instead of the $1,500 per child ($4,500) that applies to MFJ AGI of $80,001–$100,000. That overstated taxable income by $1,500, at $53,801." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied a flat rate to roughly $85k of gross wages after only the standard deduction. It ignored the 401(k) and IRA reductions to AGI and left out the $4,500 NC child deduction, and its $2,780 is higher than 3.99% of any reasonable base." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted only the $25,500 standard deduction from AGI of $82,301 and left out the NC child deduction of $1,500 for each of three children. It taxed $56,801 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 FLSA overtime premium to reach AGI of $74,555, even though the overtime deduction does not lower the AGI that NC starts from. It also left out the $4,500 NC child deduction." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It removed the $7,746 overtime premium from AGI ($74,555), but NC starts from federal AGI of $82,300.52. That understated AGI pushed it into the $2,000-per-child tier instead of the correct $1,500-per-child tier." -us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It counted the $7,746 overtime premium twice, which gave AGI of $90,047. It also left out the $4,500 NC child deduction and used 4.25% instead of the 2026 rate of 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the overtime premium to reach AGI of $74,555 instead of $82,300.52. It then used $2,500 per child, which does not match either its own AGI tier ($2,000) or the correct tier ($1,500 per child, $4,500 total)." -us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used gross wages of $85,209 as federal AGI, leaving out the traditional 401(k) ($2,778) and IRA ($130) reductions. It also left out the $4,500 NC child deduction." -us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It assumed personal exemptions for a family of five reduce the tax to about zero, but NC has no personal exemptions. Only the $25,500 standard deduction and $4,500 child deduction apply, which leaves $52,300.52 taxable." -us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 overtime premium from AGI ($74,555 instead of $82,300.52) and left out the NC child deduction of $1,500 per child." -us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the overtime premium on top of wages that already include it ($92,955), ignored the 401(k) and IRA reductions, and left out the $4,500 NC child deduction. It also used a 4.5% rate instead of 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reached taxable income of $81,509 without applying the $25,500 NC standard deduction or the $4,500 NC child deduction. It also used a 4.5% rate instead of 3.99% and subtracted a $19.41 nonrefundable credit that does not exist." +us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It used a $500-per-child deduction, which is the $120k–$140k AGI tier. At $82,301 AGI, joint filers get $1,500 per child. It also submitted $2,882, which does not match its own figure of $55,301 × 3.99% = $2,207. The correct result is $52,300 × 3.99% = $2,086.79." +us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It used the federal $29,200 standard deduction instead of NC's $25,500 joint deduction and applied the old 4.75% rate instead of 3.99%. It also invented a dependent credit instead of the $1,500-per-child deduction. Its final $5,238 does not follow from any of its own steps." +us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It claimed NC deductions wipe out taxable income. In fact, the $25,500 standard deduction and $4,500 child deduction together remove only $30,000 of the $82,301 AGI. That leaves $52,300 taxed at 3.99%, or $2,086.79." +us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It left the $130 traditional IRA deduction out of AGI. It used $2,500 per child, the $40k–$60k AGI tier, when the $80k–$100k tier gives $1,500. Its final $3,041 contradicts its own interim tax figures of about $1,972–$2,101." +us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It left out NC's child deduction entirely, which is $1,500 per child or $4,500 total at this AGI. It also failed to subtract the $130 traditional IRA contribution. As a result, it taxed $56,931 instead of $52,300." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used an outdated $21,500 joint standard deduction instead of $25,500. It also used $3,000 per child, which applies only at AGI of $40,000 or less, instead of the $1,500 tier for $80k–$100k AGI. Together these errors gave $30,500 of deductions instead of $30,000." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 FLSA overtime premium on top of gross wages, even though wages already include it. It used 4.25% instead of the 2026 rate of 3.99% and left out the $4,500 child deduction. It then added an unexplained bump to reach $3,350." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It placed $82,301 of joint AGI in the $1,000-per-child tier, which covers $100k–$120k. AGI between $80,001 and $100,000 gets $1,500 per child, so the child deduction is $4,500, not $3,000." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of $85,209 in wages, which already include it, and got AGI of $90,047 instead of $82,301. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Its AGI and standard deduction were correct, but it said no other deductions apply. It left out NC's child deduction of $1,500 per child, or $4,500, so it taxed $56,801 instead of $52,300." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It left out the $4,500 NC child deduction for three children at $80k–$100k joint AGI. It also used a 4.0% rate instead of the 2026 rate of 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted only the $25,500 standard deduction and left out NC's child deduction of $1,500 per child, or $4,500. That overstated taxable income by $4,500." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from AGI, even though they are not part of the listed wages and do not reduce AGI. This cut AGI to $61,093 instead of $82,301. The lower AGI also moved the child deduction to the $2,000 tier instead of the correct $1,500." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no derivation. At 3.99%, $3,538 implies about $88,700 of taxable income, which is more than the household's entire $82,301 AGI. So it used an outdated higher rate and did not subtract the $25,500 standard deduction and $4,500 child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 in ESI premiums to get AGI of $61,093, but those premiums do not reduce AGI, which is $82,301. That error also put it in the $2,000-per-child tier instead of the $1,500 tier for $80k–$100k AGI." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation. $3,538.77 equals about 4.75% of $74,500, so it reflects an outdated higher rate instead of 3.99% on $52,300. It also did not subtract both the $25,500 standard deduction and the $4,500 child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a 3.74% rate instead of the 2026 rate of 3.99%. It invented a $130 spousal IRA deduction for the head, who has no IRA contribution listed. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used $4,000 per child, which is above the $3,000 maximum in any tier. At $82,301 joint AGI the correct amount is $1,500 per child, or $4,500 total, not $12,000." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It claimed deductions and credits wipe out NC tax. The $25,500 standard deduction and $4,500 child deduction leave $52,300 taxable, which produces $2,086.79 at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It gave no specific derivation. Its $4,875 is about 5.25% of the $92,955 it would get by double-counting overtime, which reflects an obsolete rate with no NC deductions. The correct calculation is 3.99% of $52,300, after $25,500 standard and $4,500 child deductions from $82,301 AGI." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"$1,957.29 is 3.99% of $49,055. That means it subtracted the $7,746 overtime premium from AGI to get $74,555, but the premium is part of AGI. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"$2,027 is 3.99% of $50,801, which means it used $2,000 per child. That is the $60k–$80k AGI tier, but $82,301 of joint AGI falls in the $80k–$100k tier at $1,500 per child." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed $56,931, which means it left the $130 traditional IRA out of AGI. It also left out the $4,500 NC child deduction for three children." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It used $1,000 per child, the $100k–$120k AGI tier. At $82,301 of joint AGI the deduction is $1,500 per child, or $4,500 instead of $3,000." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It taxed roughly the full $85k of wages minus only the standard deduction. It did not subtract the $2,908 of traditional 401(k)/IRA contributions or the $4,500 NC child deduction. $2,780 also implies a rate above 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It subtracted only the $25,500 standard deduction and left out NC's child deduction of $1,500 per child, or $4,500. As a result, it taxed $56,801 instead of $52,300." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 FLSA overtime premium from AGI to get $74,555, but the premium is part of AGI. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 overtime premium to get AGI of $74,555 instead of $82,301. That moved it into the $2,000-per-child tier instead of the correct $1,500 tier for $80k–$100k AGI." +us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of wages, which already include it, and got AGI of $90,047. It used 4.25% instead of the 2026 rate of 3.99% and left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,746 overtime premium to get AGI of $74,555, but the premium is part of AGI. It also used $2,500 per child, the $40k–$60k tier, instead of $1,500 for $82,301 of joint AGI." +us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It gave no answer and no explanation for state_income_tax_before_refundable_credits. +us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It treated gross wages of $85,209 as AGI without subtracting the $2,778 traditional 401(k) and $130 IRA contributions. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used personal exemptions for a family of five, but NC has none. It returned zero tax even though $82,301 of AGI minus $30,000 in deductions leaves $52,300 taxable at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Its $47,801 of taxable income means it used $3,000 per child, which applies only at joint AGI of $40,000 or less. At $82,301 of AGI the correct amount is $1,500 per child, or $4,500 total." +us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It cut AGI to $74,555 by subtracting the $7,746 overtime premium, but the premium is part of AGI, which is $82,301. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of wages, which already include it. It did not subtract the traditional 401(k) or IRA contributions, used an outdated 4.5% rate, and left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It used the federal standard deduction instead of NC's $25,500 joint deduction and applied an outdated 4.5% rate instead of 3.99%. It left out the $4,500 child deduction and invented $19.41 in nonrefundable credits." us,scenario_026,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output, violating the required structured-output contract." us,scenario_027,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model correctly derived zero taxable income, zero gross tax, and no nonrefundable credits, but submitted -3195 instead of the zero produced by its calculation. Federal income tax before refundable credits does not become negative here, and its explanation explicitly ends with a value inconsistent with its own derivation." @@ -2333,52 +2370,53 @@ us,scenario_027,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F us,scenario_027,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly saw that CT BBCE removes the asset test, but it still applied the 100% FPL net income test and denied the household. BBCE categorical eligibility also exempts the household from the net income test. As a categorically eligible two-person household, it gets the $24/month minimum allotment ($288/year)." -us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model applied the regular elderly/disabled 100% FPL net income test (about $1,763) and concluded 'ineligible'. It never applied Connecticut's TANF non-cash BBCE, which removes that test. A categorically eligible two-person household with a $0 formula benefit receives the $24/month minimum allotment." -us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model claimed $2,513/month exceeds an approximately $2,900 limit, which contradicts its own numbers. It also applied a $3,500 elderly/disabled asset limit. Under Connecticut BBCE, gross income at 1.44× poverty is categorically eligible with no asset test, so the household receives the $24/month minimum allotment ($288/year)." -us,scenario_027,snap,claude-haiku-5.5,llm_error,asset_resource,False,"The model denied SNAP based on the $48,000 in bank assets. Connecticut BBCE through TANF non-cash eligibility waives the asset test. The household is categorically eligible and gets the $24/month two-person minimum allotment." -us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model applied a roughly $4,500 elderly/disabled resource limit to the $48,000 in assets. Connecticut's BBCE waives the SNAP asset test for this household, whose gross income is 1.44× poverty. As a result, the household qualifies for the $24/month minimum allotment." -us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model treated the roughly $4,500 elderly/disabled resource limit as binding. Connecticut BBCE categorical eligibility removes the asset test. The $0 formula benefit (30% × $2,118 = $636 > $546) is replaced by the $24/month minimum allotment for two-person households." -us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,The model denied benefits because income exceeds the two-person net income limit. That test doesn't apply to a household that is categorically eligible through Connecticut's TANF non-cash BBCE. Such a household with a $0 formula benefit receives the $24/month minimum allotment ($288/year). -us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model compared net income of about $2,308 to the roughly $1,763 net income limit and stopped there. Connecticut BBCE makes the household categorically eligible, which exempts it from the net income test. The model also omitted the two-person minimum allotment of $24/month." -us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model correctly found that gross income passes. It then applied a $4,500 elderly/disabled asset limit and denied on assets. Connecticut BBCE through TANF non-cash eligibility waives the asset test, so the household gets the $24/month minimum allotment ($288/year)." -us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model applied a $4,250 elderly/disabled resource limit and called the household ineligible. Connecticut BBCE categorical eligibility removes the asset test entirely. A categorically eligible two-person household receives at least the $24/month minimum allotment." -us,scenario_027,snap,claude-sonnet-5.5,llm_error,period_annualization,False,"The model treated the $650 annual medical and OTC expenses as a monthly amount. That produced a $615/month excess medical deduction instead of about $19 and pushed net income down to about $1,712. With the correct deductions, 30% of net income ($636) exceeds the $546 maximum, so only the $24/month minimum allotment applies, for $288/year." -us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"The model applied a $4,250 elderly/disabled asset limit to the $48,000 in bank assets. Connecticut BBCE waives the asset test for this household, whose gross income is 1.44× poverty. The household is categorically eligible for the $24/month minimum allotment." -us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model saw that gross income passes 200% FPL but then applied a 100% FPL net income test (about $1,703) as disqualifying. BBCE categorical eligibility exempts the household from the net income test. A $0 formula benefit for a categorically eligible two-person household is replaced by the $24/month minimum." -us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model concluded that net income exceeds the two-person net income limit. It never recognized Connecticut's TANF non-cash BBCE, which makes the household categorically eligible and exempt from that test. The household therefore receives the $24/month minimum allotment." -us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test ($1,762.50) to net income of about $2,313 and returned $0. Connecticut BBCE categorical eligibility removes the net income test. The two-person minimum allotment of $24/month then applies, for $288/year." -us,scenario_027,snap,gemini-3-flash-preview,llm_error,other,False,"The model correctly found BBCE eligibility and a $0 formula benefit (30% of net income exceeds the $546 maximum). It then left out the SNAP minimum allotment. Eligible one- and two-person households receive $24/month (8% of the $298 one-person maximum), or $288/year." -us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model denied SNAP based on the $48,000 in assets. Connecticut BBCE through TANF non-cash eligibility waives the resource test. The household is categorically eligible for the $24/month two-person minimum allotment." -us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,other,False,"The model recognized categorical eligibility and the $0 formula benefit, then returned $0. Categorically eligible one- and two-person households are guaranteed the minimum allotment of $24/month in FY2026, which totals $288/year." -us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum-allotment rule but used the FY2025 minimum of $23. For 2026 the minimum is 8% of the $298 one-person maximum allotment, which rounds to $24/month, giving $288/year rather than $276." -us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave no derivation, and its $0 matches either denying on the asset or income tests or stopping at the $0 formula benefit. Connecticut BBCE makes the household categorically eligible regardless of the $48,000 in assets. The model also omitted the $24/month minimum allotment for two-person households." -us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly found BBCE categorical eligibility and applied the minimum allotment, but at the FY2025 value of $23/month. The 2026 minimum is $24/month (8% of the $298 one-person maximum), so the annual total is $288, not $276." -us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $23/month minimum allotment, which is the FY2025 figure. The 2026 minimum is 8% of the $298 one-person maximum, rounded to $24/month, which gives $288 annually." -us,scenario_027,snap,gemini-3.8-flash,llm_error,asset_resource,False,"The model treated the $48,000 in assets as exceeding SNAP limits and the formula zero as final. Connecticut BBCE waives the asset test. Categorically eligible two-person households receive the $24/month minimum allotment even when 30% of net income exceeds the $546 maximum." -us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no SNAP value or explanation, so there was no answer to score." -us,scenario_027,snap,glm-5.3,llm_error,other,False,"The model correctly computed that 30% of net income exceeds the two-person maximum allotment, but it stopped at a $0 benefit. The household is categorically eligible through Connecticut BBCE, so it receives the $24/month minimum allotment ($288/year)." -us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model denied SNAP on the $48,000 in assets and on income under 'typical' rules. Connecticut BBCE through TANF non-cash eligibility waives the asset test, and gross income at 1.44× poverty qualifies. The household gets the $24/month minimum allotment." -us,scenario_027,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to $0 because shelter and rent inputs were missing. It never worked out that Connecticut BBCE makes the household categorically eligible. Under that eligibility, the $0 formula benefit is replaced by the $24/month two-person minimum allotment." -us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model said net income is above the SNAP limit for an elderly/disabled two-person household and returned $0. Connecticut BBCE categorical eligibility exempts the household from the net income test. The model also missed the $24/month minimum allotment. -us,scenario_027,snap,gpt-5.6-luna,llm_error,other,False,"The model stopped at the $0 formula benefit after standard and medical deductions. A categorically eligible two-person household receives the $24/month minimum allotment, which totals $288/year." -us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model denied SNAP because net income is above the two-person net income limit. Connecticut BBCE through TANF non-cash eligibility makes the household categorically eligible, which removes that test. The household then receives the $24/month minimum allotment." -us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model applied the elderly/disabled resource limit to the $48,000 in assets. Connecticut BBCE waives the asset test. The household is categorically eligible for the $24/month minimum allotment ($288/year)." -us,scenario_027,snap,gpt-6-luna,llm_error,other,False,"The model correctly found no positive formula allotment, but it treated $0 as final. Categorically eligible two-person households get the minimum allotment of $24/month (8% of the $298 one-person maximum), which totals $288/year." -us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,The model applied the two-person net income limit as disqualifying. Under Connecticut's TANF non-cash BBCE the household is categorically eligible and exempt from that test. Its benefit is the $24/month minimum allotment. -us,scenario_027,snap,gpt-6.1-sol,llm_error,other,False,"The model correctly found that the benefit reduction (30% × $2,118 = $636) exceeds the $546 maximum, then stopped at $0. It omitted the minimum allotment, which gives a categorically eligible two-person household $24/month." -us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model treated the $48,000 in assets as exceeding SNAP limits. Connecticut BBCE removes the asset test, and gross income at 1.44× poverty qualifies. The household is categorically eligible for the $24/month minimum allotment." -us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model applied an approximately $4,500 elderly/disabled resource limit 'regardless of income'. Connecticut BBCE categorical eligibility waives the resource test. This two-person household therefore receives the $24/month minimum allotment." -us,scenario_027,snap,grok-4.6,llm_error,other,False,"The model correctly found that 30% of net income exceeds the two-person maximum, then returned $0. It omitted the minimum allotment of $24/month that categorically eligible two-person households receive." -us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum-allotment rule but used the FY2025 minimum of $23/month. The 2026 minimum is 8% of the $298 one-person maximum, rounded to $24/month, giving $288/year." -us,scenario_027,snap,grok-build-0.1,llm_error,other,False,"The model compared annual net income (about $30,000) directly with the annual maximum allotment instead of subtracting 30% of net income from the maximum. It also omitted the $24/month minimum allotment that applies to a categorically eligible two-person household." -us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test (about $1,762) to net income of about $2,317 and called the household ineligible. Connecticut BBCE categorical eligibility removes that test. The household receives the $24/month minimum allotment." -us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model applied the 100% FPL net income limit as disqualifying because there are no shelter costs. Under Connecticut's TANF non-cash BBCE the household is categorically eligible and exempt from the net income test. Its $0 formula benefit becomes the $24/month two-person minimum. -us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model denied SNAP because net income exceeds the two-person net income limit. Connecticut BBCE makes the household categorically eligible, which waives that test. The model also missed the $24/month minimum allotment." -us,scenario_027,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model used about $30,120 as the 200% FPL gross limit for two people; the real limit is about $42,300/year, and $30,435 is only 1.44× poverty. It also applied a $4,250 asset limit, which Connecticut BBCE waives. The household is categorically eligible for the $24/month minimum allotment." -us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model applied the poverty-guideline net income test and the formula zero, then returned $0. Connecticut BBCE categorical eligibility exempts the household from the net income test. Two-person categorically eligible households receive the $24/month minimum allotment." -us,scenario_027,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model claimed that $2,536/month exceeds the gross income limit, but that is 1.44× poverty and under Connecticut's BBCE limit. It also applied a resource limit that BBCE waives. The household is categorically eligible for the $24/month minimum allotment." -us,scenario_027,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a $51,000 resource limit and said net income is below the poverty guideline, when net income of $2,118 is 1.2× poverty. With 30% × $2,118 = $636 exceeding the $546 maximum, the formula benefit is $0, and only the $24/month minimum allotment applies. The correct total is $288/year, not $5,160." +us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly found that CT's BBCE drops the asset test, but it then applied the 100% FPL net income test as a separate bar. Categorical eligibility waives that test. The eligible two-person household gets the $24/month minimum allotment even though the formula benefit is zero, which is $288 a year." +us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income of about $2,308 is above the 100% FPL net limit. It never recognized that CT's BBCE, through TANF non-cash benefits, waives the net income test. As a categorically eligible two-person household it receives the $24/month minimum allotment, or $288 a year." +us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It applied a $3,500 federal asset limit to the $48,000 in assets, but CT's BBCE has no asset test. It also wrongly said $2,513/month is above a roughly $2,900 gross limit. Gross income is under CT's 200% FPL BBCE limit, so the household is eligible and gets the $24/month minimum allotment." +us,scenario_027,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It denied SNAP because $48,000 is above the federal resource limit. Connecticut's TANF-based BBCE removes the asset test for households under 200% FPL gross income. The household is eligible and gets the $24/month minimum allotment, which is $288 a year." +us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"It applied the roughly $4,500 elderly/disabled resource limit, but CT's broad-based categorical eligibility waives the asset test. With gross income at 1.44 times FPL the household is eligible, and its zero formula benefit is raised to the $24/month two-person minimum." +us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"It treated the $48,000 in assets as disqualifying under the elderly/disabled resource limit. It missed that CT's BBCE, through TANF non-cash benefits, removes the asset test. The household is categorically eligible and receives the $24/month minimum allotment." +us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It treated the net income limit as disqualifying. Under CT's BBCE the net income test is waived. The household is eligible, and because the expected contribution of $636 is above the $546 maximum, the $24/month two-person minimum allotment applies." +us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It compared net income of about $2,308 with the roughly $1,763 net limit and gave $0. CT's BBCE makes the household categorically eligible and waives that test, so the $24/month minimum allotment for one- and two-person households applies, which is $288 a year." +us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"It computed the income steps but then denied SNAP because assets are above a $4,500 elderly/disabled limit. It missed that CT's BBCE has no asset test. The categorically eligible household gets the $24/month minimum allotment even though its formula benefit is zero." +us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"It applied a $4,250 resource limit to $48,000 in assets. Connecticut's TANF-based BBCE removes the asset test because gross income is under 200% FPL. The household is eligible and receives the $24/month minimum allotment, or $288 a year." +us,scenario_027,snap,claude-sonnet-5.5,llm_error,period_annualization,False,"It treated the $650 of annual medical and OTC costs as a monthly amount and deducted $615 a month. The correct deduction is $650/12 minus $35, about $19. That error pushed net income down to $1,712 and produced a $32/month formula benefit. Using the reference's net income of $2,118, the contribution is $636, which exceeds the $546 maximum, so only the $24/month minimum applies, or $288 a year." +us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It denied SNAP because assets are above a $4,250 elderly/disabled limit. Connecticut's BBCE waives the asset test for this household, whose gross income is 1.44 times FPL. The eligible two-person household receives the $24/month minimum allotment." +us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It correctly passed the 200% FPL gross test, but then applied the 100% FPL net income test, which CT's categorical eligibility waives. The household is eligible, and its zero formula benefit is raised to the $24/month two-person minimum, which is $288 a year." +us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It treated the net income limit as disqualifying, but CT's BBCE makes the household categorically eligible without a net income test. The $24/month minimum allotment for eligible two-person households applies, which is $288 a year." +us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It compared net income of $2,313 with the $1,762.50 limit and returned $0. It missed that CT's BBCE waives the net income test. The categorically eligible two-person household gets the $24/month minimum allotment, or $288 a year." +us,scenario_027,snap,gemini-3-flash-preview,llm_error,other,False,"It correctly identified CT BBCE eligibility and a zero formula benefit, but it stopped at the formula. Eligible one- and two-person households receive the $24/month minimum allotment, which is $288 a year." +us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"It denied SNAP because of the $48,000 in assets. Connecticut's BBCE removes the asset test for households under 200% FPL gross income. The household is eligible and receives the $24/month minimum allotment." +us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,other,False,"It recognized categorical eligibility and that 30% of net income is more than the $546 maximum, then gave $0. It left out the minimum allotment rule: eligible two-person households get $24/month, or $288 a year." +us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly applied BBCE and the minimum allotment, but it used the FY2025 minimum of $23/month. The FY2026 minimum is $24/month, so the 2026 total is $288, not $276." +us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,other,False,"It gave $0 without stating a rule. The correct answer: the household is categorically eligible under CT's BBCE, net income is $2,118, and the $636 contribution exceeds the $546 maximum, so the $24/month minimum allotment applies. Its zero amounts to dropping that minimum benefit." +us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly applied CT's BBCE and the minimum allotment, but it used the FY2025 minimum of $23. The FY2026 two-person minimum is $24/month, which gives $288 a year." +us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly applied the minimum allotment for a categorically eligible household, but it used the outdated $23/month figure. The FY2026 minimum is $24/month, which totals $288." +us,scenario_027,snap,gemini-3.8-flash,llm_error,asset_resource,False,"It treated $48,000 in assets as disqualifying and the zero formula benefit as final. CT's BBCE waives the asset test, and the eligible two-person household receives the $24/month minimum allotment." +us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"It returned no usable SNAP value. The correct derivation gives categorical eligibility under CT BBCE and the $24/month minimum allotment, which is $288 a year." +us,scenario_027,snap,glm-5.3,llm_error,other,False,"It correctly found that 30% of net income is more than the two-person maximum, but it treated the zero formula benefit as final. A categorically eligible two-person household receives the $24/month minimum allotment, which is $288 a year." +us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"It cited the $48,000 in assets and the income as disqualifying. CT's BBCE has no asset test and waives the net income test. The household receives the $24/month minimum allotment." +us,scenario_027,snap,gpt-5.4-nano,llm_error,other,False,"It defaulted to $0 because inputs were missing and computed nothing. The given facts are enough: the household is categorically eligible under CT BBCE, and its $636 contribution exceeds the $546 maximum, so the $24/month minimum allotment applies." +us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It treated the net income limit and benefit phase-out as ending eligibility. CT's BBCE waives the net income test, and an eligible two-person household whose formula benefit is zero still gets the $24/month minimum allotment." +us,scenario_027,snap,gpt-5.6-luna,llm_error,other,False,"It computed a zero formula benefit after deductions and stopped. It never applied the minimum allotment, which gives a categorically eligible two-person household $24/month, or $288 a year." +us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,It treated net income above the two-person limit as disqualifying. CT's TANF-based BBCE waives the net income test. The household is eligible and receives the $24/month minimum allotment. +us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"It applied the elderly/disabled resource limit to $48,000 in assets. Connecticut's BBCE eliminates the asset test for this household, which then receives the $24/month two-person minimum allotment." +us,scenario_027,snap,gpt-6-luna,llm_error,other,False,It concluded that high countable income produces no positive allotment and missed the floor. Categorically eligible one- and two-person households receive the $24/month minimum allotment even when the formula benefit is zero. +us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It treated income above the two-person net income limit as disqualifying. CT's BBCE waives that test, so the household is eligible and receives the $24/month minimum allotment, which is $288 a year." +us,scenario_027,snap,gpt-6.1-sol,llm_error,other,False,"It correctly found that the benefit reduction is more than the two-person maximum. It then left out the minimum allotment, which gives the categorically eligible household $24/month." +us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,It cited assets above typical SNAP limits. Connecticut's BBCE has no asset test for households under 200% FPL gross income. The eligible household receives the $24/month minimum allotment. +us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"It applied a roughly $4,500 elderly/disabled resource limit and called the household ineligible regardless of income. CT's BBCE waives the asset test, and the household receives the $24/month minimum allotment." +us,scenario_027,snap,grok-4.6,llm_error,other,False,"It correctly found that 30% of net income is more than the two-person maximum, but it reported $0. It left out the $24/month minimum allotment for eligible one- and two-person households." +us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"It correctly applied BBCE's waiver of the asset test and the minimum allotment, but it used the FY2025 minimum of $23. The FY2026 minimum is $24/month, so the total is $288." +us,scenario_027,snap,grok-build-0.1,llm_error,other,False,"It compared annual net income directly with the annual maximum allotment and gave $0, without applying the 30% contribution rate or the benefit floor. The correct derivation gives a $636 monthly contribution against the $546 maximum, so the $24/month minimum allotment applies." +us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income test and found the household ineligible. CT's BBCE waives the net income test, so the household gets the $24/month two-person minimum allotment." +us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It treated the 100% FPL net income limit as disqualifying. CT's TANF-based BBCE waives that test. The eligible two-person household receives the $24/month minimum allotment, which is $288 a year." +us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income is above the two-person limit. It missed CT's BBCE, which removes the net income test, and the $24/month minimum allotment." +us,scenario_027,snap,minimax-m3,llm_error,thresholds_rates,False,"It put the 200% FPL gross limit for two people at about $30,120, but that limit is $42,300/yr ($3,525/mo). It also applied a $4,250 asset limit that CT's BBCE waives. The household is categorically eligible and receives the $24/month minimum allotment." +us,scenario_027,snap,muse-wonder,llm_error,categorical_eligibility,False,"It applied the elderly/disabled net income limit as disqualifying. CT's BBCE waives the net income test, and the eligible household receives the $24/month two-person minimum allotment." +us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"It failed the household on the net income test and gave $0. CT's BBCE waives that test, and an eligible two-person household whose formula benefit is zero gets the $24/month minimum allotment." +us,scenario_027,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed $2,536/month is above both the gross and net limits and that assets are above the resource limit. Gross income is 1.44 times FPL, under CT's 200% BBCE limit, and BBCE waives both the net income test and the asset test. The household receives the $24/month minimum allotment." +us,scenario_027,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It made up a $51,000 resource limit, claimed net income is under the poverty line, and gave a near-maximum benefit of $430/month. Net income is $2,118, about 1.2 times FPL, so the $636 contribution exceeds the $546 maximum and only the $24/month minimum applies, which is $288 a year." us,scenario_027,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated Medicaid MAGI as counting only taxable Social Security and put countable income at about $275. Medicaid MAGI adds back non-taxable Social Security benefits, so household MAGI is the full $30,435, or 1.41x FPL. That is above the 138% expansion limit, and the model itself noted that $30,435 exceeded its own ~$28,850 threshold." us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It treated blindness and disability as categorical Medicaid eligibility and looked only at the spouse's own zero income. It also asserted that the $48,000 in assets satisfies the ABD test. The spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources far exceed the SSI couple limits ($3,000 resource limit), so no disability category applies. The adult MAGI path also fails at 1.41x FPL against the 138% limit." @@ -2527,48 +2565,49 @@ us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error, us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA expired, used a 15% bracket and a $1,000-per-child CTC, and even left out the personal exemptions its own expiration premise requires. 2026 law uses the $24,150 HoH deduction and 10%/12% brackets, giving $3,949.20 of tax that the $6,600 CTC fully offsets." us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there was no answer to score against the $0 reference." us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It used a $2,000-per-child CTC instead of $2,200 and subtracted $5,100 of refundable ACTC first, leaving only $900 as nonrefundable. The nonrefundable CTC is applied first up to the full tax liability, which zeroes out the roughly $3,950 tax." -us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work found an ACTC of about $2,665 plus an EITC of about $251, but it submitted $5,100, which is just the $1,700 × 3 refundable cap. That cap does not apply here, because the ACTC is limited to the $2,650.80 of CTC left after the $3,949.20 tax. Its EITC also used an $8,046 maximum and a $23,000 phaseout start instead of $8,231 and $23,890, which gives $624.13." -us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It started the 3-child EITC phaseout at $23,000 instead of the 2026 HoH threshold of $23,890, so its EITC came out at $436.70 instead of $624.13. It also overstated pre-credit tax at $3,955 instead of $3,949.20, which made its ACTC $2,644.80 instead of $2,650.80." -us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It wrongly put non-taxable child support into income and said no EITC applies. In fact the 3-child EITC at $60,010 AGI is still $624.13. Its $3,850 ACTC is an arbitrary figure: the ACTC is limited to the $6,600 CTC minus the $3,949.20 tax, which is $2,650.80, not the $5,100 cap or a 15%-of-earnings figure." -us,scenario_028,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It said the tax absorbs the whole CTC, but pre-credit tax is only $3,949.20 against a $6,600 CTC, which leaves $2,650.80 refundable. It also said there is no EITC, but the 3-child EITC does not fully phase out until about $62,975, so $624.13 remains at $60,010." -us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per-child CTC ($6,000 total) instead of $2,200 ($6,600), so its unused CTC came out at $2,037 instead of $2,650.80. It also set EITC to zero using an outdated ~$59,899 phaseout endpoint. Under 2026 parameters the credit is $8,231 − 21.06% × ($60,010 − $23,890) = $624.13." -us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $6,000 CTC, overstated tax at about $4,800 (the actual figure is $3,949.20), and set EITC to zero. Its own work gave an ACTC of about $1,200, yet it submitted an unrelated $4,500. The correct total is the $2,650.80 of unused CTC from $6,600, plus a $624.13 EITC." -us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,other,False,"It paid the full $1,700 × 3 = $5,100 refundable cap and never limited the ACTC to the CTC left after the nonrefundable part offsets tax, which is $6,600 − $3,949.20 = $2,650.80. It also wrongly dropped the $624.13 EITC as if income were above the 3-child phaseout." -us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per-child CTC, a $22,500 standard deduction, and 2025 brackets, so it got $1,830 of unused CTC instead of $6,600 − $3,949.20 = $2,650.80. It also put the end of the 3-child EITC phaseout at about $58,000, when the 2026 endpoint is about $62,975, which leaves a $624.13 EITC." -us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It assumed EITC ends at about $56,838 and set it to zero, even though the 2026 3-child phaseout leaves $624.13 at $60,010. Its $2,000 ACTC is an unsupported estimate; the actual ACTC is $6,600 CTC minus $3,949.20 tax, which is $2,650.80." -us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $2,000 per-child CTC and a $4,170 tax, so its ACTC was $1,830 instead of the 2026 OBBBA result: $6,600 − $3,949.20 = $2,650.80. Its EITC of $165 is too low; 2026 parameters ($8,231 maximum, $23,890 start, 21.06%) give $624.13." -us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It said tax liability exceeds the CTC, but pre-credit tax is only $3,949.20 against a $6,600 CTC, which leaves $2,650.80 refundable. It also wrongly treated the EITC as fully phased out at $60,000, when $624.13 remains under the 2026 3-child phaseout." -us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,other,False,"It claimed the nonrefundable CTC was fully used against tax and so set the refundable CTC to zero. In fact the $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, and that amount is refundable within the $5,100 cap. Its EITC of $662.87 also overshoots the correct $624.13." -us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $2,000 per-child CTC ($6,000) instead of OBBBA's 2026 $2,200 ($6,600) and overstated tax at $4,073.20, so its ACTC was $1,926.80 instead of $2,650.80. Its EITC used an $8,271 maximum and a $24,024 threshold instead of $8,231 and $23,890, which gave $692.35 instead of $624.13." -us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated the CTC as having reverted to the post-TCJA amount (about $1,000 per child), so only $66 was left over. Under OBBBA the 2026 CTC is $2,200 per child, which leaves $2,650.80 refundable after the $3,949.20 tax. Its EITC of $524 is also below the correct $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It gave zero refundable credits without computing anything. The $6,600 CTC exceeds the $3,949.20 tax, which leaves a $2,650.80 refundable CTC, and the 3-child EITC still pays $624.13 at $60,010 AGI." -us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It claimed the CTC was fully used against tax and set the refundable CTC to zero. But the $6,600 CTC minus the $3,949.20 tax leaves $2,650.80, which is refundable within the $1,700-per-child cap. Its EITC of $574 also falls short of $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, as if TCJA had expired) instead of OBBBA's $2,200 per child, and overstated tax at $3,039 (the correct figure is $3,949.20). That left no unused CTC, when the actual unused amount is $2,650.80. It also wrongly said the household is over the EITC limit, but $624.13 of EITC remains at $60,010." -us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It said phaseouts and tax liability wipe out all refundable credits. But the $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, which is refundable, and the 3-child EITC does not fully phase out until about $62,975, which leaves $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of OBBBA's 2026 $6,600 and overstated tax at $4,089.20, so its ACTC was $1,910.80 instead of $2,650.80. It also left out the $624.13 EITC that still remains in the 3-child phaseout range." -us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It wrongly said income is above the EITC phaseout endpoint (about $62,975 for three children in 2026) and that the CTC is fully absorbed. The correct figures are a $624.13 EITC and a $2,650.80 refundable CTC ($6,600 − $3,949.20)." -us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $1,838.80 ACTC reflects a $6,000 pre-OBBBA CTC and an overstated tax. The 2026 $2,200-per-child credit gives $6,600 − $3,949.20 = $2,650.80. Its EITC of $327.51 is also too low; the 2026 parameters give $8,231 − 21.06% × ($60,010 − $23,890) = $624.13." -us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no usable answer for federal_refundable_credits. The correct figure is the $2,650.80 refundable CTC plus the $624.13 EITC." -us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,other,False,"It assumed an $1,800 refundable amount per child and paid the full $5,400, ignoring that the ACTC is limited to the CTC left after tax: $6,600 − $3,949.20 = $2,650.80, under a $1,700 cap. It also wrongly zeroed out the EITC, which is $624.13 at $60,010 for three children in 2026." -us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It treated the whole $2,000-per-child CTC as refundable. In fact the refundable portion is the CTC left after offsetting tax, $6,600 − $3,949.20 = $2,650.80, capped at $1,700 per child. It also left out the $624.13 EITC." -us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed no refundable credits apply at this income. But the $6,600 CTC exceeds the $3,949.20 tax and leaves $2,650.80 refundable, and the 3-child EITC still pays $624.13 at $60,010 AGI." -us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It understated pre-credit tax at $2,184.20; the correct figure is $3,949.20, from $35,860 of taxable income after the $24,150 HoH standard deduction. That inflated the unused CTC to $4,415.80 instead of $2,650.80, and it then rounded that to $4,444 without any basis. It also dropped the $624.13 EITC." -us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It got the $2,650.80 refundable CTC right but wrongly said EITC is unavailable. The 2026 3-child EITC phases out from $23,890 at 21.06% and leaves $624.13 at $60,010, so the total is $3,274.93." -us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It paid the full $1,700 × 3 = $5,100 refundable cap without limiting the ACTC to the CTC left after the $3,949.20 tax ($2,650.80). It also left out the $624.13 EITC." -us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Its refundable CTC of about $2,651 is correct, but it put the EITC at about $650 instead of computing $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That overstated the total by about $26." -us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It computed the $2,650.80 refundable CTC correctly but left out the EITC. With three children the credit has not fully phased out at $60,010, and $624.13 remains under 2026 parameters." -us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It said no refundable credits qualify at this income. The 3-child EITC pays $624.13 at $60,010, and the $6,600 CTC exceeds the $3,949.20 tax, which leaves a $2,650.80 refundable CTC." -us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It found only about $40 of unused CTC, which means it used a much smaller CTC or a much larger tax than the correct $6,600 CTC and $3,949.20 tax. Those give a $2,650.80 refundable CTC. Its EITC of about $660 also overshoots $624.13." -us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It found only about $48 of unused CTC, when the 2026 OBBBA $6,600 CTC minus the $3,949.20 tax leaves $2,650.80. Its EITC used an $8,239 maximum and a $23,910 threshold, which gave $636 instead of $624.13." -us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,other,False,"It made a subtraction error: $6,600 minus its own $3,948 tax is $2,652, not the $2,952 it reported, so its ACTC is overstated by $300. Its EITC of $661 also overshoots the correct $624.13." -us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the CTC reverts to $1,000 per child ($3,000), which is then fully used against tax. Under OBBBA the 2026 CTC is $2,200 per child, which leaves $2,650.80 refundable after the $3,949.20 tax. Its EITC used an $8,302 maximum and a $24,099 threshold, which gave $739 instead of $624.13." -us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"Its refundable CTC of $2,649 is essentially correct, but it put the EITC at about $340 instead of $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That understated the total by about $285." -us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no usable answer for federal_refundable_credits. The correct figure is the $2,650.80 refundable CTC plus the $624.13 EITC." -us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $624.13 is exactly right, but it understated pre-credit tax at $3,838.10. The correct tax is $3,949.20: $35,860 of taxable income after the $24,150 HoH deduction, with 10% on the first $17,700 and 12% above. That inflated the ACTC to $2,761.90 instead of $2,650.80." -us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,other,False,"It took the ACTC as the lesser of the 15% earnings formula and the $5,100 cap, and skipped the limit to the CTC left after tax: $6,600 − $3,949.20 = $2,650.80. It also put the end of the EITC phaseout at about $61,500 and below $60,000 income, when the 2026 3-child endpoint is about $62,975, which leaves $624.13." -us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Its refundable CTC of $2,650.80 is correct, but it started the EITC phaseout at $22,210 with an $8,234 maximum instead of the 2026 HoH $23,890 and $8,231. That gave $273.32 instead of $624.13." -us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $2,000 per-child CTC ($6,000) and a $4,081.20 tax, so its ACTC was $1,918.80 instead of the OBBBA result: $6,600 − $3,949.20 = $2,650.80. It also inflated older EITC parameters ad hoc to a $8,222 maximum and a $25,914 threshold, which gave $1,042 instead of $624.13." -us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It said the 15%-of-earnings-over-$2,500 limit binds, but that limit is $8,625 and does not bind. The binding limit is the CTC left after tax, $2,650.80, and its $3,200 figure does not come from either limit. It also left out the $624.13 EITC." +us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Its own working reached $2,665 ACTC plus about $251 EITC (≈$2,916), but it submitted $5,100, which is just the $1,700 × 3 refundable cap. It also understated the EITC by using a $8,046 maximum and a $23,000 phase-out start instead of $8,231 and $23,890, which give $624.13." +us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The ACTC (~$2,645) was close to right, but the EITC phase-out used a $23,000 starting point instead of the 2026 $23,890 threshold. That gave $436.70 instead of $8,231 − 21.06% × $36,120 = $624.13." +us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It counted $1,267 of child support as income and wrongly concluded the EITC was fully phased out; the EITC is $624.13 at $60,010 AGI. Its $3,850 ACTC is an invented number that ignores the actual binding limit: $6,600 CTC minus $3,949.20 of tax = $2,650.80." +us,scenario_028,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It claimed tax liability absorbs the whole CTC. In fact, pre-credit tax of $3,949.20 is well below the $6,600 CTC, leaving $2,650.80 refundable. It also wrongly zeroed the EITC, which is still $624.13 at $60,010 for three children." +us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) instead of $2,200 ($6,600), so the unused refundable amount came out as $2,037 instead of $2,650.80. It also treated the EITC as fully phased out above ~$59,899, missing the $624.13 credit that remains under the 2026 $23,890 phase-out start." +us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Its reasoning overstated pre-credit tax at ~$4,800 and derived about $1,200 of ACTC, then submitted an unrelated $4,500. The correct figures are $3,949.20 of tax against a $6,600 CTC ($2,650.80 refundable) plus a $624.13 EITC it wrongly zeroed." +us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It paid the full $1,700-per-child refundable cap without limiting the ACTC to the CTC left after offsetting $3,949.20 of tax ($6,600 − $3,949.20 = $2,650.80). It also wrongly zeroed the $624.13 EITC." +us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC, a $22,500 HoH standard deduction and old brackets, giving $4,170 of tax and $1,830 ACTC. The correct inputs are $2,200 per child and a $24,150 deduction, giving $3,949.20 of tax and $2,650.80 ACTC. It also set the EITC to $0 using a stale ~$58,000 phase-out end, although the 2026 three-child credit is still $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It never computed the unused CTC: $6,600 − $3,949.20 = $2,650.80. It guessed $2,000 based on a $2,000-per-child framework, and it wrongly zeroed the EITC using a ~$56,838 phase-out end instead of the 2026 parameters that leave $624.13." +us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the $2,000-per-child CTC with an overstated $4,170 tax, giving $1,830 ACTC instead of $6,600 − $3,949.20 = $2,650.80. It also understated the EITC at ~$165 instead of $8,231 − 21.06% × ($60,010 − $23,890) = $624.13." +us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It asserted that tax liability exceeds the CTC. In fact, the $3,949.20 pre-credit tax is below the $6,600 CTC, leaving $2,650.80 refundable, and the EITC is $624.13, not phased out, at $60,010." +us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"It claimed the CTC is fully used against tax and so dropped the $2,650.80 refundable CTC ($6,600 − $3,949.20). Its EITC of $662.87 also overstates the $624.13 that the $8,231 maximum and $23,890 phase-out start produce." +us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of $6,600 and overstated tax at $4,073.20, giving $1,926.80 ACTC instead of $2,650.80. Its EITC parameters ($8,271 maximum, $24,024 threshold) produced $692.35 instead of $624.13." +us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It left only $66 of unused CTC, which grossly overstates tax relative to the CTC. Pre-credit tax is $3,949.20 against a $6,600 CTC, leaving $2,650.80. Its $524 EITC also falls short of the correct $624.13." +us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It assumed income rules out all refundable credits. The CTC of $6,600 exceeds the $3,949.20 tax by $2,650.80, which is refundable, and the three-child EITC is still $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It wrongly concluded the CTC is fully used against tax and so dropped the $2,650.80 unused refundable portion. It also slightly understated the EITC at $574 instead of $624.13." +us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used the pre-TCJA $1,000-per-child CTC ($3,000) instead of the 2026 $2,200 per child ($6,600), so it found no unused CTC. It also wrongly treated $60,010 as above the three-child EITC limit, when the credit is still $624.13." +us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It zeroed all refundable credits without computing that the $6,600 CTC exceeds the $3,949.20 tax by $2,650.80. It also missed the $624.13 EITC that remains at $60,010 with three children." +us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) instead of $6,600, giving $1,910.80 ACTC instead of $2,650.80. It left out the $624.13 EITC entirely." +us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It assumed the CTC is fully absorbed and the EITC fully phased out. The $3,949.20 tax leaves $2,650.80 of the $6,600 CTC refundable, and the EITC is $624.13 because the three-child phase-out does not end until about $62,970." +us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $1,838.80 ACTC reflects a $2,000-per-child CTC and overstated tax; the correct figure is $6,600 − $3,949.20 = $2,650.80. Its $327.51 EITC understates the $624.13 that the 2026 $8,231 maximum and $23,890 threshold produce." +us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so there was no answer to score against the $3,274.93 reference." +us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used an $1,800 per-child refundable cap instead of $1,700 and ignored the limit of the CTC left after offsetting $3,949.20 of tax, which is $2,650.80. It also wrongly zeroed the $624.13 EITC." +us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It treated the whole $2,000-per-child CTC as refundable. In fact, only the $2,650.80 not absorbed by the $3,949.20 tax is refundable, under the $1,700-per-child cap, and the $624.13 EITC is left out." +us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed no refundable credits apply. The $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, which is refundable, and the three-child EITC is still $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It understated pre-credit tax at $2,184.20 instead of $3,949.20 ($35,860 taxable income after the $24,150 HoH deduction), which inflated the unused CTC to ~$4,416. It also wrongly zeroed the $624.13 EITC." +us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It computed the $2,650.80 refundable CTC correctly but declared the EITC unavailable. The three-child EITC is still $8,231 − 21.06% × ($60,010 − $23,890) = $624.13." +us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It paid the full $1,700-per-child refundable cap without limiting the ACTC to the $2,650.80 of CTC left after the $3,949.20 tax offset. It also left out the $624.13 EITC." +us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The ACTC of $2,651 was right, but the EITC was rounded to ~$650 instead of being computed exactly: $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That overstated the total by about $26." +us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It computed the $2,650.80 refundable CTC exactly but left out the $624.13 EITC, which is still available at $60,010 with three children." +us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It claimed no refundable credits qualify. The $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, which is refundable, and the three-child EITC is $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It left only ~$40 of unused CTC, which implies pre-credit tax near the full credit. Tax is actually $3,949.20 against a $6,600 CTC, leaving $2,650.80. Its ~$660 EITC also overstates the $624.13." +us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its EITC of $636 was close, but it found only $48 of unused CTC, overstating pre-credit tax relative to the CTC. The correct refundable CTC is $6,600 − $3,949.20 = $2,650.80." +us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $6,900 total CTC ($2,300 per child) instead of $6,600 ($2,200 per child), giving $2,952 ACTC instead of $2,650.80. Its $661 EITC also overstates the $624.13." +us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the CTC reverted to $1,000 per child ($3,000), which is fully absorbed by tax. OBBBA sets $2,200 per child, so $2,650.80 of the $6,600 CTC is refundable. Its $739 EITC also used an inflated $8,302 maximum instead of $8,231." +us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The ACTC (~$2,649) was right, but the EITC was understated at ~$340. The 2026 parameters ($8,231 maximum, 21.06% phase-out from $23,890) give $624.13." +us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so there was no answer to score against the $3,274.93 reference." +us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The EITC was exactly right at $624.13, but pre-credit tax was understated at $3,838.10 instead of $3,949.20. The correct tax is 10% of $17,700 plus 12% of $18,160, on $35,860 taxable income after the $24,150 HoH deduction. That error inflated the ACTC to $2,761.90 instead of $2,650.80." +us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It set the ACTC to the $5,100 cap without limiting it to the CTC left after the $3,949.20 tax offset ($2,650.80). It also wrongly treated the EITC as fully phased out at $60,000 when it is $624.13." +us,scenario_028,federal_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"The ~$2,650 ACTC was right, but the EITC was rounded to ~$650 instead of being computed exactly: $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. The total of $3,300 overshoots." +us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The refundable CTC of $2,650.80 was right, but the EITC phase-out started at $22,210 instead of the 2026 $23,890 threshold (with an $8,234 maximum). That gave $273.32 instead of $624.13." +us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC ($6,000) and $4,081.20 of tax, giving $1,918.80 ACTC instead of $2,650.80. It also inflated the EITC to $1,042 with an invented $25,914 phase-out start and $8,222 maximum instead of $23,890 and $8,231, which give $624.13." +us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed the earned-income formula limits the ACTC to $3,200. That formula allows $8,625; the binding limit is the $2,650.80 of the $6,600 CTC left after the $3,949.20 tax. It also left out the $624.13 EITC." us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -2634,50 +2673,51 @@ us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_erro us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model treated PA tax as a federal-style calculation with a standard deduction and graduated rates. PA has no standard deduction and taxes all income at a flat 3.07%. Its $3,480 implies about $113,000 of income taxed at 3.07%, or an effective rate of about 5.8% on $60,010, either of which is wrong. The correct tax is $1,842.31." us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model subtracted a made-up $17,800 PA head-of-household standard deduction. PA has no standard deduction, so 3.07% applies to the full $60,010, giving $1,842.31 instead of $1,296." us,scenario_028,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model correctly computed $1,842.31 and then subtracted a made-up $1.55 PA Child and Dependent Care credit. The household has no dependent care expenses, so no such credit exists. Because PA's dependent care credit is refundable, it would not reduce tax before refundable credits in any case." -us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said PA has no state EITC and only the nonrefundable Tax Forgiveness program. It missed the refundable Working Pennsylvanians Tax Credit, which pays 10% of the federal EITC. With the $624.13 federal EITC, that credit is $62.41." -us,scenario_028,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model checked only the PA child and dependent care credit and stopped at $0 because there were no care expenses. It missed PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed PA's EIC is nonrefundable. PA's Working Pennsylvanians Tax Credit is a refundable credit equal to 10% of the federal EITC, which gives $62.41 for this household's $624.13 federal EITC." -us,scenario_028,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model concluded that no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC. This three-child household receives $624.13 of federal EITC, so the PA credit is $62.41." -us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated Tax Forgiveness as the only possible PA refundable credit and ruled it out on income. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41 here." -us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model limited its analysis to Tax Forgiveness and asserted PA has no other refundable credits. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It overlooked the Working Pennsylvanians Tax Credit, a refundable credit of 10% of the federal EITC. With the $624.13 federal EITC, the credit is $62.41." -us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated that Pennsylvania has no state EITC. The Working Pennsylvanians Tax Credit is a refundable state EITC set at 10% of the federal EITC, so this household with a $624.13 federal EITC gets $62.41." -us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said PA has no refundable credit comparable to a state EITC and looked only at Tax Forgiveness. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41 on this household's $624.13 federal EITC." -us,scenario_028,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model assumed no PA refundable credit is available at this income level. The Working Pennsylvanians Tax Credit is 10% of any positive federal EITC, and the household's $624.13 federal EITC yields $62.41." -us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model tested only Tax Forgiveness income thresholds and also miscounted the family as four people. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said PA provides no refundable credit for this household. It missed the Working Pennsylvanians Tax Credit, a refundable credit of 10% of the federal EITC, which comes to $62.41 here." -us,scenario_028,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model claimed PA offers no refundable individual income tax credits. That is outdated: the Working Pennsylvanians Tax Credit is a refundable 10%-of-federal-EITC credit, which equals $62.41 for this household." -us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model found no applicable PA refundable credits for 2026. It missed the Working Pennsylvanians Tax Credit, which equals 10% of the federal EITC; applied to the $624.13 federal EITC, that is $62.41." -us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no state refundable credits apply. It omitted PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model checked only the PA Child and Dependent Care Enhancement credit, which requires care expenses. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model concluded no PA refundable credit is available. It missed the Working Pennsylvanians Tax Credit, which pays 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said no state refundable credits apply. It ignored PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC and equals $62.41 here." -us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model said the household does not qualify for PA refundable credits. Any household with a positive federal EITC qualifies for the Working Pennsylvanians Tax Credit at 10% of that amount, so this household gets $62.41." -us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said the household does not qualify for refundable PA credits. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model found the household ineligible for any PA refundable credit. It overlooked the Working Pennsylvanians Tax Credit, a refundable credit worth 10% of the federal EITC, which is $62.41 here." -us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there was no answer to score. The correct value is 10% of the $624.13 federal EITC under the Working Pennsylvanians Tax Credit, or $62.41." -us,scenario_028,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model treated the Property Tax/Rent Rebate, which requires age 65 or disability, as the only candidate. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no PA refundable credit indicated by the household facts. It missed the Working Pennsylvanians Tax Credit, which follows directly from the household's $624.13 federal EITC and comes to $62.41 at 10%." -us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed income was too high for any refundable credit. The household still receives a $624.13 federal EITC, and PA's refundable Working Pennsylvanians Tax Credit pays 10% of it, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for child care expenses and other special inputs, found none, and returned $0. It missed the Working Pennsylvanians Tax Credit, which needs only a federal EITC and equals 10% of the $624.13 federal credit, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It omitted the Working Pennsylvanians Tax Credit, a refundable credit of 10% of the federal EITC, which is $62.41 here." -us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model assumed no PA refundable credit applies at this income. The household still has a $624.13 federal EITC, and the Working Pennsylvanians Tax Credit pays 10% of it, or $62.41." -us,scenario_028,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model correctly applied PA's 10%-of-federal-EITC refundable credit but rounded the federal EITC to about $650. The exact 2026 three-child phase-out leaves $624.13 at roughly $60,010 of income, so the PA credit is $62.41, not $65." -us,scenario_028,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit is indicated. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model asserted there are no state refundable credits. It missed PA's refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said no PA refundable credits apply. It overlooked the Working Pennsylvanians Tax Credit, which pays 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Pennsylvania has no applicable refundable credits. The Working Pennsylvanians Tax Credit is a refundable 10%-of-federal-EITC credit, which is $62.41 for this household." -us,scenario_028,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model asserted PA has no refundable earned income credit and looked only at Tax Forgiveness. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model stated that PA has no state EITC. The Working Pennsylvanians Tax Credit is a refundable state EITC at 10% of the federal credit, so this household gets $62.41." -us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model considered only Tax Forgiveness and rejected it on income. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model tested only Tax Forgiveness thresholds and also miscounted the family as four people. It missed the refundable Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model claimed PA has no state EITC. The Working Pennsylvanians Tax Credit is a refundable credit equal to 10% of the federal EITC, which gives $62.41 on this household's $624.13 federal EITC." -us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model said none of PA's refundable credits apply. It missed the Working Pennsylvanians Tax Credit, which is 10% of the $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model checked only Tax Forgiveness eligibility. It never applied the refundable Working Pennsylvanians Tax Credit, which is 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model stated that PA has no refundable state EITC. The Working Pennsylvanians Tax Credit is a refundable credit at 10% of the federal EITC, worth $62.41 for this household." -us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said no PA refundable credits apply. It missed the Working Pennsylvanians Tax Credit, which is 10% of the federal EITC, or $62.41 on the $624.13 federal credit." +us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said PA has no state EITC and offers only nonrefundable Tax Forgiveness. It missed the refundable Working Pennsylvanians Tax Credit, which pays 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model only checked the PA child and dependent care credit, which is $0 with no child-care expenses. It missed the refundable PA state EITC, which pays 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed the PA earned income credit is nonrefundable. The Working Pennsylvanians Tax Credit is refundable at 10% of the federal EITC, so this household's $624.13 federal EITC produces a $62.41 refundable credit." +us,scenario_028,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model found no applicable refundable PA credit. It missed PA's refundable state EITC at 10% of the federal EITC, which is $62.41 on this household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness, which this income rules out, and concluded no PA refundable credit applies. It missed the refundable Working Pennsylvanians Tax Credit at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model treated Tax Forgiveness as PA's only relief mechanism and said no other refundable credits exist. It missed PA's refundable state EITC, equal to 10% of the federal EITC ($624.13), which gives $62.41." +us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said PA has no applicable refundable credits. It missed the Working Pennsylvanians Tax Credit, a refundable 10% match of the household's $624.13 federal EITC, which equals $62.41." +us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said outright that Pennsylvania has no state EITC. PA's refundable EITC equals 10% of the federal credit, so the $624.13 federal EITC for this three-child head of household yields $62.41." +us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said PA has no state EITC and analyzed only the nonrefundable Tax Forgiveness credit. It missed the refundable Working Pennsylvanians Tax Credit at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model concluded no PA refundable credit applies. It missed PA's refundable state EITC, which is 10% of the household's $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model said PA offers no refundable credit at this income. The federal three-child EITC has not fully phased out at about $60,010 of AGI ($624.13), so PA's 10% refundable EITC match pays $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model tested only the Tax Forgiveness income thresholds. It missed the refundable PA state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model concluded PA provides no refundable credit for this household. It missed the Working Pennsylvanians Tax Credit, which refunds 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model claimed PA has no refundable individual income tax credits. That ignores the refundable PA state EITC at 10% of the federal EITC, which is $62.41 here." +us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model said no PA refundable credits apply for 2026. PA's refundable EITC equals 10% of the federal EITC, and the household's $624.13 federal EITC produces $62.41." +us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model found no state refundable credits. It missed PA's refundable Working Pennsylvanians Tax Credit at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model checked only the PA child and dependent care enhancement credit, which is $0 with no child-care costs. It missed the refundable PA state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It missed PA's refundable state EITC, a 10% match of the $624.13 federal EITC, which is $62.41." +us,scenario_028,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,The model found no state refundable credits. PA's refundable EITC at 10% of the household's $624.13 federal EITC gives $62.41. +us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model concluded the household does not qualify for PA refundable credits. It missed the Working Pennsylvanians Tax Credit at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model concluded no PA refundable credit applies. It missed PA's refundable state EITC, which is 10% of the federal EITC ($624.13) for a three-child head of household near $60k AGI, or $62.41." +us,scenario_028,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model said the household is ineligible for any PA refundable credit. It missed PA's refundable state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for state_refundable_credits. The correct answer is PA's refundable state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model looked only at the Property Tax/Rent Rebate, which requires age 65+ or disability, and concluded no refundable credit applies. It missed PA's refundable state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no indicated PA refundable credits. PA's refundable EITC pays 10% of the federal EITC, and this household's $624.13 federal EITC produces $62.41." +us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed income was too high for any major refundable credit. At about $60,010 of AGI with three children, the federal EITC is still $624.13, so PA's 10% refundable match pays $62.41." +us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model linked PA refundable credits only to child-care expenses. It missed the refundable PA state EITC, 10% of the $624.13 federal EITC, which needs only earned income and qualifying children and equals $62.41 here." +us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model concluded no PA refundable credit applies. It missed the Working Pennsylvanians Tax Credit at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model concluded no PA refundable credit applies to these facts. It missed PA's refundable state EITC at 10% of the household's $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model treated this income as too high for any PA refundable credit. The three-child federal EITC is still $624.13 at about $60,010 of AGI, so PA's 10% refundable EITC is $62.41." +us,scenario_028,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model correctly used PA's refundable EITC at 10% of the federal EITC, but rounded the federal EITC to about $650. Computing the three-child head-of-household phase-out exactly at about $60,010 of AGI gives a federal EITC of $624.13, so the PA credit is $62.41, not $65." +us,scenario_028,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model found no indicated PA refundable credit. It missed PA's refundable state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model asserted there are no state refundable credits. PA's refundable EITC pays 10% of the household's $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said no PA refundable personal income tax credits apply. It missed the Working Pennsylvanians Tax Credit at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model said PA has no applicable refundable credits. PA's refundable state EITC equals 10% of the federal EITC, which is $62.41 on this household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,The model said PA has no refundable earned income credit and looked only at nonrefundable Tax Forgiveness. PA's refundable EITC at 10% of the $624.13 federal EITC gives $62.41. +us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said explicitly that PA has no state EITC. The Working Pennsylvanians Tax Credit is a refundable 10% match of the federal EITC, which is $62.41 on $624.13." +us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model treated nonrefundable Tax Forgiveness as the only PA credit to test. It missed the refundable PA state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model checked only the Tax Forgiveness income limits. It missed PA's refundable state EITC, which pays 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model claimed PA has no state EITC. PA's refundable Working Pennsylvanians Tax Credit pays 10% of the federal EITC, which is $62.41 on this household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model found no applicable PA refundable credits. It missed the refundable PA state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"The model said there is no state EITC amount for PA. PA's refundable EITC equals 10% of the household's $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness eligibility at $60,010 of income. It missed the refundable PA state EITC at 10% of the $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model said PA has no refundable EITC. The Working Pennsylvanians Tax Credit is a refundable 10% match of the federal EITC, which is $62.41 on the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model concluded no PA refundable credits apply. It missed PA's refundable state EITC at 10% of the $624.13 federal EITC, or $62.41." us,scenario_028,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_029,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It stated ""AGI is $50,312 ($50,000 taxable financial assistance + $312 taxable interest),"" treating the non-taxable financial-assistance input as ordinary taxable income when that variable enters no AGI component, leaving actual AGI at $312 of taxable interest alone. It compounded this by applying a pre-TCJA 2026 schedule — an $8,600 single standard deduction with 10%/15% brackets — instead of the 2026 single standard deduction of about $16,100 under the permanent OBBBA rates. Either correction alone puts the $312 of interest below the standard deduction, yielding $0 rather than $5,676.80." us,scenario_029,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It explicitly ""treated [the $50,000 financial assistance] as taxable income for federal tax purposes"" and added it to the $312 of interest, but financial assistance is a non-taxable receipt that contributes nothing to AGI, so the return reports only $312 of taxable interest. Its $5,236 is consistent with running a bracket schedule on roughly $45,700 of taxable income after subtracting a standard deduction from a $50,312 base. The correct derivation is $312 AGI minus the 2026 single standard deduction of about $16,100, giving $0 taxable income and $0 tax before refundable credits." @@ -2707,51 +2747,52 @@ us,scenario_029,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model skip us,scenario_029,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model built Oklahoma taxable income of $50,312 by adding the $50,000 of financial assistance to the $312 of taxable interest, but financial assistance is a non-taxable transfer that is excluded from federal AGI, which is Oklahoma's starting point, so Oklahoma AGI is $312. It compounded this with an $8,600 standard deduction in place of Oklahoma's single standard deduction plus the $1,000 personal exemption, then ran the graduated 0.25%–4.75% schedule over $41,712 of income that does not exist; with only $312 of interest, Oklahoma taxable income is zero and the tax is $0.00." us,scenario_029,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no arithmetic, but $1,827 is what Oklahoma's graduated schedule yields on roughly $43,000 of taxable income — $50,312 of gross income less about $7,350 of standard deduction and personal exemption — which is reachable only by treating the $50,000 of financial assistance as taxable income. Financial assistance is non-taxable and never enters federal AGI or Oklahoma AGI; the sole taxable item is $312 of interest, which falls below Oklahoma's standard deduction and produces $0.00 of tax." us,scenario_029,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model's $36,662 base is exactly $50,312 minus a $13,650 deduction, showing it counted the non-taxable $50,000 financial assistance as Oklahoma taxable income and then subtracted a federal-style standard deduction rather than Oklahoma's single standard deduction plus $1,000 personal exemption. Excluding the non-taxable assistance leaves $312 of taxable interest, below every Oklahoma deduction and filing threshold, so the correct liability is $0.00 instead of $1,160." -us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Correctly identified the $40-per-exemption Sales Tax Relief Credit and its $20,000 limit, but added the $50,000 of financial assistance to gross household income. Gross income for the credit is only the $312 of taxable interest, which is far below $20,000, so one exemption yields $40." -us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Wrongly claimed that Oklahoma has no refundable individual credits, which misses the refundable Sales Tax Relief Credit. That credit pays $40 per exemption to this filer, whose gross income of $312 is under the $20,000 limit." -us,scenario_029,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Treated earned income as a requirement for every Oklahoma refundable credit. The Sales Tax Relief Credit does not require earned income, and it pays $40 per exemption when gross income ($312 here) is at or below $20,000." -us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Disqualified the filer from the Sales Tax Relief Credit by counting the $50,000 of financial assistance (and citing assets) against the $20,000 income limit. The credit's gross income is only the $312 of interest, and the credit has no asset test, so the filer gets $40 for one exemption." -us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Named the Sales Tax Relief Credit but denied it because there was no earned income and the filer had large assets and financial assistance. The credit has no earned-income or asset test, and the filer's $312 gross income is under $20,000, so it pays $40." -us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"Assumed take-up of the Sales Tax Relief Credit was zero and that its thresholds were not met, even though the prompt says to assume take-up. With $312 of gross income, the filer is under the $20,000 limit and receives $40 for one exemption." -us,scenario_029,state_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,Assumed Oklahoma refundable credits require earned income or dependents. The Sales Tax Relief Credit requires neither and pays $40 per exemption to a filer with $312 of gross income. -us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Checked only the Oklahoma EITC and never considered the refundable Sales Tax Relief Credit. That credit pays $40 per exemption because the filer's gross income ($312) is below $20,000." -us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Wrongly asserted that Oklahoma has no general refundable credit for this filer, assuming relief requires elderly, disabled, or dependent status. Those statuses only raise the income limit to $50,000. Under the base $20,000 limit, the Sales Tax Relief Credit pays $40 on $312 of gross income." -us,scenario_029,state_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Wrongly concluded that no refundable credits apply at this income level. The very low $312 income is exactly what qualifies the filer for the $40-per-exemption Sales Tax Relief Credit under its $20,000 limit." -us,scenario_029,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,Treated zero tax liability as ruling out refundable credits. The Sales Tax Relief Credit is refundable regardless of liability and pays $40 to a single filer with $312 of gross income. -us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,Considered only the Oklahoma EITC (5% of a $0 federal EITC) and wrongly said no other refundable credits apply. The Sales Tax Relief Credit gives $40 for the filer's one exemption. -us,scenario_029,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Looked only at the EITC-based credit and missed the Sales Tax Relief Credit. That credit pays $40 per exemption when gross income ($312) is at or below $20,000." -us,scenario_029,state_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,Tied all Oklahoma refundable credits to earned income and the federal EITC. The Sales Tax Relief Credit has no earned-income requirement and yields $40 here. -us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Computed gross household income as $50,312 by adding the $50,000 of financial assistance. The credit's gross income is only the $312 of interest, which is under the $20,000 limit, so the credit is $40." -us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Wrongly asserted that no state refundable credits apply and missed the Sales Tax Relief Credit. That credit pays $40 per exemption because gross income of $312 is below $20,000." -us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Included the $50,000 of financial assistance in gross household income for the Sales Tax Relief Credit test. Gross income is only the $312 of interest, which is under $20,000, so the credit is $40 for one exemption." -us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"Wrongly concluded that the household qualifies for no refundable credits, overlooking the Sales Tax Relief Credit. That credit pays $40 per exemption to filers with gross income of $20,000 or less; this filer has $312." -us,scenario_029,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Gave $0 without any derivation and never applied the Sales Tax Relief Credit. That credit pays $40 for one exemption because gross income of $312 is under the $20,000 limit." -us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,age_disability,False,"Wrongly believed the Sales Tax Relief Credit requires the filer to be 65+, disabled, or have dependents. Those conditions only raise the limit to $50,000; every filer qualifies under the base $20,000 limit, and the filer's $312 income meets it for $40." -us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Wrongly concluded that no refundable Oklahoma credit applies and missed the Sales Tax Relief Credit. That credit gives $40 per exemption when gross income ($312) is at or below $20,000." -us,scenario_029,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Checked only the Oklahoma EITC (zero because the federal EITC is zero) and ignored the Sales Tax Relief Credit, which pays $40 for this filer's one exemption." -us,scenario_029,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"Wrongly asserted that Oklahoma offers no refundable credit to this household. The Sales Tax Relief Credit pays $40 per exemption because gross income of $312 is below $20,000." -us,scenario_029,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"Gave $0 without considering the Sales Tax Relief Credit. That credit pays $40 for this single filer's one exemption, given gross income of $312." -us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Wrongly concluded that the facts support no refundable credit. The listed $312 of interest income, below the $20,000 limit, establishes eligibility for the $40 Sales Tax Relief Credit." -us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to $0 for lack of explicit eligibility indicators. The Sales Tax Relief Credit's only conditions are an adult filer and gross income under $20,000, both met here, and it yields $40." -us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,Assumed earned income was the basis for any Oklahoma refundable credit. The Sales Tax Relief Credit pays $40 per exemption on gross income of $312 regardless of earnings. -us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Required earned income or a dependent for an Oklahoma refundable credit. The Sales Tax Relief Credit needs neither and gives $40 because gross income is below $20,000." -us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,Required earned income or dependents for a refundable credit and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a single filer with $312 of gross income. -us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Wrongly concluded that no facts establish eligibility. The filer's adult age and $312 of gross income (under $20,000) qualify them for the $40 Sales Tax Relief Credit." -us,scenario_029,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance toward the Sales Tax Relief Credit's income limit. Gross income for the credit is only the $312 of interest, so the filer qualifies for $40." -us,scenario_029,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"Required children or earned income for a refundable credit and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to any adult filer with gross income of $20,000 or less." -us,scenario_029,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"Required earnings or children for eligibility and ignored the Sales Tax Relief Credit. That credit pays $40 for one exemption because gross income ($312) is below $20,000." -us,scenario_029,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"Required earned income, dependents, or specific expenses for a refundable credit. The Sales Tax Relief Credit depends only on gross income at or below $20,000 and pays $40 here." -us,scenario_029,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,Assumed qualifying children were needed and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a childless single filer with $312 of gross income. -us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance and investment assets against the Sales Tax Relief Credit limits. The credit has no asset test, and its gross income is only the $312 of interest, so the filer gets $40." -us,scenario_029,state_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Included the $50,000 of financial assistance in gross household income for the Sales Tax Relief Credit. Gross income is $312, which is below the $20,000 limit, so the credit is $40." -us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"Wrongly asserted that no refundable state credits apply, overlooking the Sales Tax Relief Credit. That credit pays $40 per exemption because gross income ($312) is under $20,000." -us,scenario_029,state_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Concluded that income or assets block the Oklahoma low-income credit. The Sales Tax Relief Credit has no asset test, and the filer's gross income of $312 is far below the $20,000 limit, giving $40." -us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"Treated being under 65 with no disability or dependents as disqualifying for the Sales Tax Relief Credit. Those factors only raise the limit to $50,000; under the base $20,000 limit, the filer's $312 income qualifies for $40." -us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Added the $50,000 of financial assistance to gross household income and found the filer over the sales-tax-relief limit. Counted gross income is only the $312 of interest, so the credit is $40 for one exemption." -us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Falsely claimed that Oklahoma has no state income tax. Oklahoma levies an income tax and has a refundable Sales Tax Relief Credit, which pays this filer $40 on $312 of gross income." -us,scenario_029,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"Named the Sales Tax Relief Credit but treated it as requiring earned income. It requires only gross income at or below $20,000, which $312 meets, so it pays $40." -us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,Considered only the EITC and child-related credits and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a single filer whose gross income is $312. -us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"Wrongly concluded that no refundable credits are indicated. The filer's $312 gross income, below the $20,000 limit, qualifies them for the $40-per-exemption Sales Tax Relief Credit." +us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It identified the Sales Tax Relief Credit and the $20,000 single-filer limit. It then added the $50,000 of financial assistance to gross income, but the credit's income measure is only the $312 of interest. At $312 the filer qualifies for $40 × 1 exemption = $40." +us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It claimed Oklahoma has no refundable individual credits, which is wrong. The refundable Sales Tax Relief Credit pays $40 per exemption to filers with gross income at or below $20,000, so a single filer with $312 of income gets $40." +us,scenario_029,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It assumed refundable credits require earned income, but the Sales Tax Relief Credit has no such requirement. It pays $40 per exemption when gross income is at or below $20,000, which gives $40 on $312 of interest." +us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It counted the $50,000 of financial assistance (and mentioned assets) against the $20,000 Sales Tax Relief Credit limit. The credit's gross income is only the $312 of interest, and assets are not tested. That yields $40 × 1 exemption = $40." +us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It ruled out the Sales Tax Relief Credit because of the assets and financial assistance and because there was no earned income. The credit has no asset test, no earned-income requirement, and its gross income here is $312, so the filer is under the $20,000 limit and gets $40." +us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"It treated take-up of the sales tax relief credit as zero, even though the prompt says to assume filing and take-up. It also said the threshold was not met, when $312 is well under the $20,000 limit, so the refundable credit is $40." +us,scenario_029,state_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,"It assumed Oklahoma refundable credits require earned income or dependents. The Sales Tax Relief Credit pays $40 per exemption to any single filer with gross income at or below $20,000, so this filer gets $40." +us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It checked only the Oklahoma EITC and never applied the refundable Sales Tax Relief Credit. That credit pays $40 per exemption when gross income ($312 here) is at or below $20,000." +us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It wrongly believed the credit requires elderly or disabled status or dependents. The refundable Sales Tax Relief Credit is open to any filer under the $20,000 gross income limit, which gives $40 for this single filer with $312 of income." +us,scenario_029,state_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said no refundable credit applies at this income level, which gets the low-income Sales Tax Relief Credit backwards. A single filer with $312 of gross income is under the $20,000 limit and receives $40 per exemption." +us,scenario_029,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It treated zero tax liability as ruling out refundable credits. The Sales Tax Relief Credit is paid regardless of liability: $40 per exemption when gross income is at or below $20,000, which gives $40 here." +us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,It checked only the Oklahoma EITC (5% of a $0 federal EITC) and missed the refundable Sales Tax Relief Credit. That credit pays $40 for one exemption when gross income is $312. +us,scenario_029,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It looked only at the EITC and left out the Sales Tax Relief Credit. That credit is $40 per exemption for gross income at or below $20,000, which gives $40 for this single filer." +us,scenario_029,state_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It tied all Oklahoma refundable credits to the federal EITC and missed the Sales Tax Relief Credit, which does not depend on earned income. That credit pays $40 when gross income of $312 is under the $20,000 limit." +us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It computed gross income as $50,312 by adding the $50,000 of financial assistance. The Sales Tax Relief Credit's gross income is just the $312 of interest, which is under the $20,000 limit and yields $40." +us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It missed the refundable Sales Tax Relief Credit, which pays $40 per exemption to a single filer with gross income at or below $20,000. That gives $40 for this household." +us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It counted the $50,000 of financial assistance toward the Sales Tax Relief Credit's $20,000 limit. The credit's gross income is only the $312 of interest, so the single filer qualifies for $40." +us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It missed the refundable Sales Tax Relief Credit, which gives $40 per exemption when gross income is at or below $20,000. With $312 of income, this filer receives $40." +us,scenario_029,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no basis for $0 and left out the Sales Tax Relief Credit. That credit is $40 × 1 exemption for a single filer whose gross income ($312) is under $20,000." +us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,age_disability,False,"It treated age 65+, disability, or dependents as a requirement for the Sales Tax Relief Credit. Those factors only raise the income limit to $50,000. A 21-year-old single filer qualifies under the $20,000 limit, and with $312 of income receives $40." +us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It missed the refundable Sales Tax Relief Credit. The credit is $40 per exemption for gross income at or below $20,000, which gives $40 on $312 of interest." +us,scenario_029,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It looked only at the Oklahoma EITC and missed the Sales Tax Relief Credit. That credit does not require earned income and pays $40 per exemption when gross income is at or below $20,000." +us,scenario_029,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"It claimed no refundable credit applies, but the Sales Tax Relief Credit pays $40 per exemption to a single filer under the $20,000 gross income limit. That gives $40 here." +us,scenario_029,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,It reported no refundable Oklahoma credits and missed the Sales Tax Relief Credit. That credit is $40 for one exemption with gross income of $312. +us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no fact supporting a refundable credit, but the only facts needed for the Sales Tax Relief Credit are residency and gross income at or below $20,000. Both are met, which gives $40." +us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It defaulted to zero for lack of explicit eligibility facts. Low gross income ($312, under $20,000) is enough for the refundable Sales Tax Relief Credit of $40 per exemption." +us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It required earned income or some other qualifying basis, but the Sales Tax Relief Credit is based only on gross income at or below $20,000. This filer qualifies for $40." +us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It assumed Oklahoma refundable credits require earned income or a dependent. The Sales Tax Relief Credit pays $40 per exemption to a single filer with gross income at or below $20,000, which gives $40." +us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It said earned income or dependents were required and missed the Sales Tax Relief Credit, which depends only on gross income at or below $20,000. That gives $40 for one exemption." +us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It found no eligibility basis, but $312 of gross income is under the $20,000 limit for the refundable Sales Tax Relief Credit. That gives $40 × 1 exemption = $40." +us,scenario_029,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It added the $50,000 of financial assistance to household income for the sales tax relief test. The credit's gross income is only the $312 of interest, which is under $20,000 and yields $40." +us,scenario_029,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"It treated qualifying children or earned income as prerequisites. The Sales Tax Relief Credit requires neither and pays $40 per exemption when gross income is at or below $20,000." +us,scenario_029,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It looked for earnings or children and missed the Sales Tax Relief Credit, which is based only on gross income ($312, under $20,000). That gives $40." +us,scenario_029,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It assumed refundable credits need earned income, dependents, or specific expenses. The Sales Tax Relief Credit pays $40 per exemption to any filer with gross income at or below $20,000." +us,scenario_029,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"It treated qualifying children as necessary for a refundable credit. A childless single filer with $312 of gross income qualifies for the $40 Sales Tax Relief Credit under the $20,000 limit." +us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It counted the $50,000 of financial assistance and the investment assets against the sales tax relief limit. The credit has no asset test and uses only the $312 of gross income, which is under $20,000 and yields $40." +us,scenario_029,state_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It added the $50,000 of financial assistance to gross household resources for the Sales Tax Relief Credit. The credit's income measure is the $312 of interest, so the filer is under $20,000 and gets $40." +us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"It missed the refundable Sales Tax Relief Credit, which pays $40 per exemption to a single filer with gross income at or below $20,000. That gives $40 here." +us,scenario_029,state_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It ruled out the low-income credit based on income and assets. The Sales Tax Relief Credit has no asset test, and its gross income here is $312, below the $20,000 limit, so it pays $40." +us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"It treated being under 65, not disabled, and without dependents as disqualifying for the Sales Tax Relief Credit. Those statuses only raise the income limit to $50,000. The base $20,000 limit applies here, and the filer's $312 qualifies for $40." +us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It counted the $50,000 of financial assistance toward the $20,000 sales tax relief limit for an under-65 filer. The credit's gross income is only the $312 of interest, which gives $40 × 1 exemption = $40." +us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It wrongly claimed Oklahoma has no state income tax. Oklahoma has an income tax and a refundable Sales Tax Relief Credit of $40 per exemption for gross income at or below $20,000, which gives $40 here." +us,scenario_029,state_refundable_credits,muse-wonder,llm_error,categorical_eligibility,False,"It required earned income or a qualifying child, but the Sales Tax Relief Credit depends only on gross income at or below $20,000. That pays $40 for this single filer." +us,scenario_029,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It named the Sales Tax Relief Credit but wrongly treated it as requiring earned income. The credit pays $40 per exemption whenever gross income ($312) is at or below $20,000." +us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It considered only EITC and child-related credits and missed the Sales Tax Relief Credit. That credit pays $40 per exemption to a single filer with gross income under $20,000." +us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It found no indicated refundable credit and missed the Sales Tax Relief Credit, which is $40 for one exemption when gross income ($312) is below the $20,000 limit." us,scenario_030,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $5,000 of educational assistance and $12,000 of financial assistance in AGI, inflating it from $13,000 to $30,000. With only $13,000 of taxable wages, the single-filer standard deduction eliminates all taxable income, so applying the 10% and 12% brackets to $14,600 was erroneous." us,scenario_030,federal_refundable_credits,claude-fable-5,llm_error,age_disability,False,"The model calculated a phased-out childless EITC without first applying the minimum-age requirement. At age 23, the filer is ineligible for the 2026 childless EITC, so the phase-in and phaseout arithmetic never applies." us,scenario_030,federal_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated the 23-year-old as eligible for the childless EITC despite the minimum-age requirement. It also mislabeled $1,940 as the childless maximum and submitted that amount even though its own 7.65% calculation produced $994.50." @@ -2780,53 +2821,54 @@ us,scenario_030,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_outpu us,scenario_030,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscalculated Medicare tax: 1.45% of $13,000 is $188.50, not $189 or $189.90. It then compounded the arithmetic error by stating that $806 plus $189 equals $995.90; the exact component sum is $994.50." us,scenario_030,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It counted income correctly ($2,083/month) but tested it against the regular 130% FPL gross limit. It never applied Texas's broad-based categorical eligibility (BBCE) limit of 165% FPL ($2,152/month), which the household passes. As a result it missed the $24/month minimum allotment that a categorically eligible one-person household receives." -us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It correctly computed net income of about $1,658 and found that 30% of net income exceeds the $298 maximum. However, it applied a net income test that BBCE waives in Texas, and it set the benefit to $0 instead of the $24/month minimum allotment for a categorically eligible one-person household." -us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It added the $5,000 of educational assistance to income, but SNAP excludes it, so the correct total is $25,000 rather than $30,000. It then applied only a 130% FPL gross limit and missed Texas's 165% BBCE limit, which the actual $2,083/month gross income passes. That would have led to the $24/month minimum allotment." -us,scenario_030,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It used only wages and returned $0 without applying any rule. It left out Texas's BBCE pathway (gross income of $2,083 is below the 165% FPL limit of $2,152) and the $24/month minimum allotment for an eligible one-person household." -us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"It identified Texas BBCE at 165% FPL but calculated the limit as about $2,072 using the outdated 2025 guideline. The 2026 limit is 165% × $1,304.17 = $2,152, which the $2,083 gross income passes. Categorical eligibility then yields the $24/month minimum allotment, not $0." -us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"It excluded the $12,000 of financial assistance, which SNAP counts as $1,000/month of unearned income, so it understated net income at $663 instead of $1,658. It also abandoned its own $99/month result and jumped to a near-maximum figure. The correct contribution of $498 exceeds the $298 maximum, which leaves only the $24 minimum." -us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It counted only the $13,000 of wages and left out the $1,000/month of financial assistance that SNAP counts as unearned income. That produced a $134/month benefit. With the assistance included, net income is $1,658, the 30% contribution ($498) exceeds the $298 maximum, and the household receives only the $24 minimum allotment." -us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It assumed the financial assistance does not count, but SNAP counts the $12,000 of financial assistance as unearned income. Only the educational assistance is excluded. Including the $1,000/month raises net income to $1,658 and the contribution to $498, so the benefit falls to the $24 minimum rather than $101/month." -us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It said Texas does NOT have broad-based categorical eligibility, which is wrong. Texas grants categorical eligibility through TANF non-cash services at a 165% FPL gross limit ($2,152/month), and the $2,083 gross income passes it. That waives the income tests and pays the $24/month minimum allotment." -us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"It left out the $1,000/month of financial assistance, which counts as unearned income. It also made an arithmetic error: $298 minus about $200 is roughly $98, not $350, and $350/month exceeds the $298 maximum allotment. With all income counted, the contribution of $498 leaves only the $24 minimum." -us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It treated the financial assistance as excluded, but SNAP counts the $12,000 as unearned income. Only the educational assistance is excluded. Including it gives net income of $1,658 and a $498 contribution above the $298 maximum, so the benefit is the $24 minimum allotment rather than $101/month." -us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, and so inflated gross income to $2,500/month instead of $2,083. It also applied the 130% gross limit instead of Texas's 165% BBCE limit of $2,152, which the correct income passes. That would have led to the $24/month minimum allotment." -us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It counted the excluded $5,000 of educational assistance, giving gross income of $30,000 instead of $25,000, and compared it to a 2025-based 130% FPL limit. Texas BBCE uses 165% of the 2026 guideline ($25,822/year), which $25,000 passes. That makes the household categorically eligible for the $24/month minimum." -us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, so its gross income exceeded even the 165% BBCE limit. It also zeroed the benefit when the contribution exceeded the maximum. With correct income of $2,083/month the household is categorically eligible, and the $24/month minimum allotment applies." -us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It counted the pregnant head as a two-person household and used the $546 two-person maximum. SNAP does not count an unborn child, so the household has one member and the maximum is $298. Its $497 contribution exceeds $298, which leaves only the $24 minimum allotment ($288/year)." -us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It correctly used a one-person household with $25,000 of countable income but tested it only against the 130% FPL limit. Texas BBCE raises the gross limit to 165% FPL ($25,822/year), which the household passes. Categorical eligibility then pays the $24/month minimum allotment." -us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It based the benefit only on the $13,000 of wages and left out the $1,000/month of financial assistance that counts as unearned income. Correct net income is $1,658, which gives a $498 contribution above the $298 maximum. That leaves only the $24 minimum, not about $217/month." -us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income limit and missed Texas's BBCE limit of 165% FPL ($2,152/month), which the $2,083 gross income passes. Categorical eligibility waives the income tests and pays the $24/month one-person minimum allotment." -us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It applied the 165% FPL BBCE limit but calculated it as $24,849 using the 2025 guideline of $15,060. The 2026 guideline is $15,650, which makes the limit $25,822.50, so the $25,000 income passes. The household is then categorically eligible for the $24/month minimum." -us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 without any derivation. Under Texas BBCE (gross income $2,083 is below the 165% FPL limit of $2,152), the household is categorically eligible. Although its $498 contribution exceeds the $298 maximum, it receives the $24/month minimum allotment." -us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It treated income above the net limit as disqualifying, but Texas BBCE waives the net income test. When the contribution exceeds the maximum allotment, a categorically eligible one-person household still receives the $24/month minimum allotment." -us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,It applied a net income threshold that Texas BBCE waives and returned $0. It missed that a categorically eligible one-person household whose $498 contribution exceeds the $298 maximum receives the $24/month minimum allotment. -us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It correctly counted the financial assistance as unearned income. However, it applied a net income threshold that BBCE waives and dropped the benefit to $0 instead of paying the $24/month minimum allotment that applies to categorically eligible one- and two-person households." -us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"It counted the unborn child to make a household of two, but SNAP counts only the pregnant head. It also deducted $8,080 of medical expenses, although the excess medical deduction applies only to elderly or disabled members. With one person and no medical deduction, the contribution of $498 exceeds the $298 maximum, which leaves the $24 minimum." -us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"It counted income correctly but applied the regular gross and net income limits. Texas BBCE uses a 165% FPL gross limit ($2,152/month), which the $2,083 income passes, and it waives the net test. The household therefore receives the $24/month minimum allotment." -us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated countable income as very low, ignoring $1,083/month of wages and $1,000/month of financial assistance. Correct net income is $1,658, and the $498 contribution exceeds the $298 maximum, so only the $24 minimum allotment is paid." -us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 without applying any SNAP rule. The household is categorically eligible under Texas BBCE because gross income of $2,083 is below the 165% FPL limit of $2,152. With a contribution above the maximum allotment, it receives the $24/month one-person minimum." -us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,It correctly found that 30% of net income exceeds the $298 one-person maximum. It then set the benefit to $0 and missed that a household categorically eligible through Texas BBCE receives the $24/month minimum allotment. -us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It ruled the household out on the gross income limit and missed that Texas BBCE sets that limit at 165% FPL ($2,152/month). The $2,083 gross income passes it, so the household is categorically eligible and receives the $24/month minimum allotment." -us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It said $25,000 exceeds the Texas gross income limit, but the Texas BBCE limit is 165% of the 2026 guideline ($25,822.50/year), which the household passes. Categorical eligibility then pays the $24/month minimum allotment." -us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It compared $25,000 to the regular gross income limit and missed Texas BBCE at 165% FPL ($25,822.50/year). Under BBCE the household is categorically eligible and receives the $24/month minimum allotment." -us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"It applied the standard SNAP income limits, which Texas BBCE replaces with a 165% FPL gross test that this household passes. It missed the $24/month minimum allotment that applies when the $498 contribution exceeds the $298 maximum." -us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,It correctly counted the financial assistance and found that the 30% contribution exceeds the $298 maximum. It then returned $0 instead of the $24/month minimum allotment owed to a categorically eligible one-person household. -us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It deducted 30% of net wages only and left out the $1,000/month of financial assistance that SNAP counts as unearned income. Correct net income is $1,658, which gives a $498 contribution above the $298 maximum and leaves only the $24 minimum allotment." -us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,It correctly found countable income too high for a positive formula benefit but ignored the minimum allotment. A one-person household categorically eligible through Texas BBCE receives $24/month even when the contribution exceeds the maximum. -us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 without any computation. The household is categorically eligible under Texas BBCE (gross income of $2,083 is below the 165% FPL limit of $2,152) and receives the $24/month minimum allotment." -us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross limit to the $25,000 of countable income and missed Texas BBCE at 165% FPL ($25,822.50/year). The household passes that limit and receives the $24/month minimum allotment." -us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"It claimed income is above the Texas 165% FPL BBCE limit, but $25,000 is below 165% × $15,650 = $25,822.50. That makes the household categorically eligible and entitles it to the $24/month minimum allotment." -us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"It disqualified the household on the 100% FPL net income test, which Texas BBCE waives for categorically eligible households. It also missed that the household receives the $24/month minimum allotment when the contribution exceeds the maximum." -us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross limit (about $20,800) and missed Texas BBCE, which raises the limit to 165% FPL ($25,822.50/year). The $25,000 income passes that limit, which makes the household eligible for the $24/month minimum allotment." -us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, and tested income against the 130% FPL limit. Correct gross income is $2,083/month, which passes Texas's 165% BBCE limit of $2,152 and yields the $24/month minimum allotment." -us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there is no answer to score. The correct derivation is BBCE categorical eligibility with a $24/month minimum allotment, or $288/year." -us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"It correctly averaged countable income at $2,083.33/month but applied the regular gross income limit. Texas BBCE sets the limit at 165% FPL ($2,152/month), so the household is categorically eligible for the $24/month minimum allotment." -us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"It compared annual net income of $10,400 against a monthly limit of $1,255, which is a unit mismatch. It also left out the $1,000/month of financial assistance. Correct monthly net income of $1,658 under Texas BBCE gives a $498 contribution above the maximum and a $24/month minimum allotment." -us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"It correctly counted $25,000 of income but tested it only against the 130% FPL limit (about $20,350). Texas BBCE uses 165% FPL ($25,822.50/year), which the household passes, and that results in the $24/month minimum allotment." -us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It counted only wages, leaving out the $1,000/month of financial assistance, and skipped the $209 standard deduction. It then dropped its own $31–32/month result and gave an unsupported $2,576. Correct net income is $1,658, which gives a contribution above the $298 maximum and the $24 minimum." -us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It counted the unborn child as a second household member and set net income to $0, ignoring $1,083/month of wages and $1,000/month of financial assistance. The SNAP unit is one person with net income of $1,658, so the benefit is the $24/month minimum, not a maximum allotment." +us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Counted income correctly at about $2,083/month but applied the 130% FPL gross test, which does not apply here. Texas BBCE (TANF non-cash categorical eligibility) raises the gross limit to 165% FPL, and the household is at 1.60×. It therefore stays eligible and gets the $24/month minimum allotment ($288/year)." +us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Correctly computed net income of about $1,658 and saw that the 30% contribution exceeds the $298 maximum. It then applied the net income test, which BBCE waives, and left out the $24 minimum allotment for eligible one-person households. That floor gives $288/year." +us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Counted educational assistance as income for a $30,000 total, although SNAP excludes it. It then tested against 130% FPL and ignored Texas BBCE's 165% limit. Countable income is $25,000 (1.60× FPL), so the household is eligible for the $24/month minimum." +us,scenario_030,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Gave no real derivation, and its $0 implies it treated the household as ineligible under standard gross/net tests. Under Texas BBCE the household passes at 1.60× FPL. Its $498 contribution exceeds the $298 maximum, so the $24 minimum applies, giving $288/year." +us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Recognized Texas BBCE at 165% FPL but put the limit at about $2,072/month. The correct figure is 1.65 × $1,304.17 = $2,152, so gross income of $2,083 (1.60× FPL) passes. The household then gets the $24 minimum allotment, not $0." +us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Excluded the $12,000 financial assistance, which PolicyEngine counts as $1,000/month of unearned income. It also contradicted its own math by computing about $99/month and then submitting close to the maximum. With the assistance counted, net income is $1,658 and the contribution is $498, which exceeds $298, so only the $24 minimum is paid." +us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Counted only wages and dropped the $12,000 financial assistance, which counts as $1,000/month of unearned income. Including it raises net income to $1,658 and the contribution to $498, above the $298 maximum. The benefit falls to the $24 minimum, not $134/month." +us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Explicitly excluded the financial assistance, but SNAP counts it as $1,000/month of unearned income. That raises net income from $658 to $1,658. The contribution of $498 then exceeds the $298 maximum, leaving the $24 minimum ($288/year)." +us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Counted income correctly at $2,083/month but wrongly said Texas has no BBCE. Texas confers categorical eligibility through TANF non-cash benefits at 165% FPL, and the household's 1.60× ratio passes. The income tests are waived, and the $24/month minimum allotment is paid." +us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"Left out the $1,000/month financial assistance from countable income. It also made an arithmetic error: $298 minus a contribution of about $200 is about $98, not $350. With the assistance counted, the contribution is $498, which exceeds the maximum, so the benefit is the $24 minimum." +us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Treated the financial assistance as excluded, but SNAP counts it as $1,000/month of unearned income. Net income is therefore $1,658, not $658, and the $498 contribution exceeds the $298 maximum. Only the $24 minimum allotment remains." +us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Counted the $5,000 educational assistance, which SNAP excludes, and applied the 130% FPL gross test instead of Texas BBCE's 165% limit. Countable gross income is $2,083/month (1.60× FPL), so the household is eligible and gets the $24 minimum." +us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Counted educational assistance for a $30,000 total, then applied the 130% FPL gross test that Texas BBCE waives. Countable income is $25,000 (1.60× FPL), under the 165% BBCE limit. The household therefore gets the $24/month minimum." +us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Wrongly counted the excluded educational assistance. It also stopped at a contribution above the maximum allotment, missing that a categorically eligible one-person household receives the $24 minimum allotment ($288/year)." +us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Counted the pregnant head as a two-person SNAP household, but an unborn child does not add to SNAP household size. With size 1 the maximum is $298, and the correctly computed $497 contribution exceeds it. The benefit is the $24 minimum, not $546 − $497 = $49." +us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Correctly used a one-person household and $25,000 countable income but applied the 130% FPL gross test. Texas BBCE uses 165% FPL, and $25,000 is 1.60× the guideline. The household is eligible and gets the $24 minimum allotment." +us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Based the benefit only on the $13,000 of earnings and left out the $12,000 financial assistance ($1,000/month unearned). Its $217/month result does not follow the standard formula. With all income counted, net income is $1,658 and the $498 contribution exceeds $298, so only the $24 minimum is paid." +us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross limit, which Texas BBCE replaces with a 165% FPL limit through TANF non-cash eligibility. Gross income is 1.60× FPL and passes. The one-person household then gets the $24 minimum allotment." +us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"Correctly invoked Texas BBCE at 165% FPL but miscomputed the limit as $24,849. The correct limit is 1.65 × $15,650 = $25,822, so $25,000 of countable income (1.60×) passes. The household is eligible for the $24/month minimum." +us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Gave no derivation; its $0 implies it applied the standard income tests or let a negative formula result stand. Under Texas BBCE the household is eligible at 1.60× FPL, and the $24 minimum allotment covers the case where the $498 contribution exceeds the $298 maximum." +us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Applied the net income limit, which BBCE categorical eligibility waives. It also missed that an eligible one-person household whose formula benefit is zero or negative still receives the $24 minimum allotment." +us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Treated net income above the threshold as disqualifying, but Texas BBCE waives the net income test. The categorically eligible one-person household receives the $24/month minimum allotment." +us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Correctly counted the financial assistance but concluded that net income above the threshold gives $0. Under BBCE the household remains eligible. With the $498 contribution above the $298 maximum, the $24 minimum allotment applies ($288/year)." +us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"Counted the unborn child to make a two-person household. It also deducted medical expenses, although that deduction is limited to elderly or disabled members and the 23-year-old head is neither. With size 1 and no medical deduction, net income is $1,658, the contribution is $498, and the benefit falls to the $24 minimum." +us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"Counted income correctly but applied the federal gross and net limits that Texas BBCE waives. Gross income is 1.60× FPL, under the 165% BBCE limit. The household is eligible and receives the $24 minimum allotment." +us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Described the household as having very low countable cash income, ignoring $1,083/month of wages and $1,000/month of financial assistance. Counting both gives net income of $1,658 and a $498 contribution above the $298 maximum. The benefit is the $24 minimum, not about $176/month." +us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Gave no substantive derivation; its $0 implies it found the household ineligible or let the formula result stand. The household is categorically eligible under Texas BBCE, and its $498 contribution exceeds the $298 maximum, so it gets the $24 minimum allotment." +us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum but stopped at $0. A household that stays eligible through Texas BBCE gets the $24 minimum allotment for one- and two-person households, which is $288/year." +us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Applied a gross income limit that Texas BBCE raises to 165% FPL. Countable gross income of $2,083/month is 1.60× the guideline and passes. The household receives the $24 minimum allotment." +us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Correctly counted the financial assistance and excluded the educational assistance, but wrongly concluded the household exceeds the Texas gross limit. The Texas BBCE limit is 165% FPL ($2,152/month), and $2,083 passes. The household gets the $24 minimum." +us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Tested $25,000 against the standard one-person gross limit instead of the 165% FPL limit under Texas BBCE through TANF non-cash eligibility. At 1.60× FPL the household passes and receives the $24 minimum allotment." +us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"Applied the federal income limits, which BBCE categorical eligibility waives. It also missed that the eligible one-person household receives the $24 minimum allotment even though its $498 contribution exceeds the $298 maximum." +us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Correctly found the 30% contribution above the one-person maximum but returned $0. A household categorically eligible under Texas BBCE gets the $24/month minimum allotment, or $288/year." +us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Applied the benefit formula to wages only and left out the $1,000/month financial assistance that SNAP counts as unearned income. Counting it raises net income to $1,658 and the contribution to $498, above the $298 maximum. Only the $24 minimum is paid." +us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"Concluded the income was too high for any allotment, missing the $24 minimum benefit. That floor applies to one- and two-person households that remain eligible, and this household is eligible through Texas BBCE." +us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"Gave no derivation; its $0 implies it treated the household as ineligible. It passes the $190 asset test and the 165% FPL BBCE gross limit at 1.60×. With the $498 contribution above the $298 maximum, the $24 minimum applies." +us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"Correctly counted $25,000 of income but applied the 130% FPL gross test. Texas BBCE replaces that test with a 165% FPL limit, which the household meets at 1.60×. The result is the $24 minimum allotment." +us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"Leaned toward counting educational assistance, which SNAP excludes, and wrongly said income exceeds the Texas BBCE 165% limit. Countable income of $25,000 is 1.60× the $15,650 guideline, under the $25,822 limit. The household is eligible for the $24 minimum." +us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"Correctly counted income and deductions but treated failing the 100% FPL net income test as disqualifying. Texas BBCE waives the net test, so the household remains eligible and receives the $24/month minimum allotment." +us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Correctly counted $25,000 of income and excluded educational assistance, but applied the 130% FPL gross limit. Under Texas BBCE the limit is 165% FPL, which the household meets at 1.60×. It gets the $24 minimum allotment." +us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"Counted the $5,000 educational assistance, which SNAP excludes, and applied the 130% FPL gross test instead of Texas BBCE's 165% limit. Countable income is $25,000 (1.60× FPL), so the household is eligible for the $24 minimum." +us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value or explanation, so no answer could be parsed. The correct output is $288: twelve months of the $24 minimum allotment." +us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"Correctly computed $2,083.33/month of countable income but applied the standard one-person gross limit. Texas BBCE uses 165% FPL ($2,152/month), which the household meets. It receives the $24 minimum allotment." +us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"Compared an annual figure ($10,400) against a monthly net limit ($1,255) and ignored both the standard deduction and the $1,000/month financial assistance. Correctly computed, monthly net income is $1,658. The household is categorically eligible under BBCE and gets the $24 minimum." +us,scenario_030,snap,muse-wonder,llm_error,categorical_eligibility,False,"Applied the net income limit, which Texas BBCE categorical eligibility waives. It also missed that an eligible one-person household gets the $24 minimum allotment when the 30% contribution exceeds the maximum." +us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly counted $25,000 but tested it against the 130% FPL gross limit instead of the 165% FPL limit under Texas BBCE through TANF non-cash eligibility. At 1.60× FPL the household is eligible for the $24 minimum." +us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Left out the $1,000/month financial assistance and the standard deduction. It then submitted $2,576, which contradicts its own $372–$384 arithmetic. Counted correctly, net income is $1,658 and the contribution is $498, above the $298 maximum, so only the $24 minimum is paid." +us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Counted the unborn child to make a two-person household and claimed net income of $0, ignoring $2,083/month of countable income. It then submitted $3,576, which does not match its own $4,272. A one-person household with $1,658 net income gets only the $24 minimum." us,scenario_030,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no TANF output, violating the required structured-output contract rather than making a substantive benefit calculation." us,scenario_030,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly counted the unborn child as a current dependent child and constructed a two-person Texas TANF assistance unit. A pregnant adult with no born dependent child does not satisfy the family-with-dependent-children requirement, so the $271 monthly grant schedule does not apply." @@ -2834,50 +2876,51 @@ us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_er us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income." us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation." us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic." -us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,state_local_rule,False,"It subtracted the $1,165 of alimony paid, which SSI-based income counting does not allow. It never applied California's $230/month Aged & Disabled income disregard. Countable income is $23,853 − $2,760 = $21,093, which is under the $22,024.80 limit (138% of the 2026 FPL)." -us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It applied SSI's $20/month ($240/year) general exclusion instead of California's $230/month ($2,760/year) disregard, which replaces it. It also used an outdated ~$21,600 limit instead of $22,024.80. With the correct disregard, countable income is $21,093, which is under the limit." -us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It used a wrong 138% FPL figure (~$18,564) and compared gross income without California's $230/month disregard. It also suggested that the $4,200 in savings fails a resource test, but California's limit for an individual is $130,000. Countable income of $21,093 is under the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated Medicare age as ruling out Medicaid and never evaluated the aged (senior-or-disabled) pathway. Being 65 or older meets that pathway's category test, and Medicare eligibility bars only the 19–64 expansion group." -us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It treated the aged pathway as MAGI-based with a 100% FPL limit. California's Aged & Disabled program uses SSI-style counting with a $230/month disregard and a 138% FPL limit ($22,024.80), and countable income of $21,093 is under that limit." -us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the limit without subtracting California's $230/month ($2,760/year) Aged & Disabled disregard. After the disregard, countable income is $21,093, under 138% of the 2026 FPL ($22,024.80)." -us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to an outdated ~$21,600 limit and skipped California's $230/month disregard. The correct comparison is $21,093 of countable income against $22,024.80, which passes." -us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It ultimately applied the MAGI expansion pathway, which does not cover a 67-year-old. It used an understated $20,783 limit and gave no disregard. The aged non-MAGI pathway subtracts $2,760/year from $23,853 and compares the resulting $21,093 to $22,024.80, which passes." -us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It excluded Social Security, which SSI-style counting includes in full. It then tested income against the SSI federal benefit rate (~$13,596) instead of the 138% FPL limit for California's Aged & Disabled program. Correct countable income is $23,853 − $2,760 = $21,093, which is under $22,024.80." -us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"It claimed the head is in no categorical group, but age 67 meets the aged category on its own. It also compared gross income to 138% FPL without California's $230/month disregard, which brings countable income to $21,093, under the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It tested gross income of $23,853 against 138% FPL and omitted California's $230/month Aged & Disabled disregard. Countable income after the disregard is $21,093, which is under $22,024.80." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It compared gross monthly income of $1,987.75 to an understated limit of $1,732; the correct limit is $1,835.40/month. It also skipped the $230/month disregard, which brings countable income to $1,757.79/month, under the limit." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It counted the full $23,853 as countable income and never subtracted California's $230/month disregard for the Aged & Disabled program. Countable income of $21,093 is under the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a limit close to the correct one but compared gross income of $23,853 with no disregard. California subtracts $230/month ($2,760/year), leaving $21,093, which is under 138% FPL." -us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied the correct 138% FPL test but used gross income of $23,853 and missed California's $230/month disregard. Countable income of $21,093 is under the ~$22,025 limit." -us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no specific reasoning; the answer is consistent with comparing gross income to the limit. The correct derivation subtracts California's $2,760 annual Aged & Disabled disregard from $23,853, leaving $21,093, which is under the $22,024.80 limit (138% FPL)." -us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It compared gross income of $23,853 to 138% FPL without California's $230/month disregard. Countable income of $21,093 passes the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It identified the correct non-MAGI 138% FPL test but used gross monthly income (~$1,988) without the $230/month disregard. After the disregard, income is ~$1,758/month, under the $1,835.40 limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no specific reasoning, and the answer implies the income test failed. The aged pathway is open at 67, and after California's $2,760 annual disregard, countable income of $21,093 is under the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It tested gross income of $23,853 and never subtracted California's $230/month Aged & Disabled disregard. That disregard brings countable income to $21,093, under 138% FPL ($22,024.80)." -us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It concluded income exceeds the non-MAGI limit without applying California's $230/month disregard. Countable income of $21,093 is under the $22,024.80 limit (138% FPL)." -us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It compared total income to 138% FPL with no deductions and missed California's $230/month disregard. Income after the disregard is $21,093, which is under $22,024.80." -us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed that Medicare eligibility at 65 or older rules out Medicaid. In PolicyEngine, Medicare eligibility bars only the 19–64 expansion group, and the aged senior-or-disabled pathway remains open and passes here." -us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It tested the head against the expansion group, which covers only ages 19–64, using a wrong FPL figure (its numbers put income at ~199% FPL). It never evaluated the aged pathway, where income after the $230/month disregard is $21,093, under 138% FPL." -us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It required disability or other qualifying flags and did not recognize that age 67 alone meets the aged category. It also never ran the income test, which passes after California's $230/month disregard ($21,093 vs. $22,024.80)." -us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the aged limit and fell back on spend-down, omitting California's $230/month disregard. With the disregard, countable income of $21,093 meets the $22,024.80 limit (138% FPL) for full-scope eligibility." -us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It concluded countable income exceeds the limit without applying California's $230/month Aged & Disabled disregard. That disregard leaves $21,093, which is under $22,024.80." -us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged limit without subtracting California's $2,760 annual disregard. Countable income of $21,093 is under 138% of the 2026 FPL ($22,024.80)." -us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It tested income against an SSI-linked income level instead of the 138% FPL limit for California's Aged & Disabled program. It also omitted the $230/month disregard; applying it gives $21,093 of countable income, under $22,024.80." -us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,state_local_rule,False,"It applied SSI's $20/month general income exclusion. California substitutes a $230/month disregard for it, which leaves $21,093, under the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged threshold without California's $230/month disregard. After the disregard, countable income of $21,093 passes the 138% FPL limit." -us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It used SSI's $20 general income exclusion instead of California's $230/month disregard, which replaces it. With the correct $2,760 annual disregard, countable income is $21,093, under $22,024.80." -us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It assumed PolicyEngine excludes Medicare-eligible people from Medicaid. Medicare eligibility bars only the 19–64 expansion group, and the aged senior-or-disabled pathway grants eligibility here." -us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It claimed $4,200 in bank assets exceeds the resource limit, but California's Aged & Disabled limit for an individual is $130,000. It also compared gross income without the $230/month disregard, which brings countable income to $21,093, under $22,024.80." -us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,state_local_rule,False,"It treated the full $23,853 as countable and missed California's $230/month Aged & Disabled disregard. Countable income of $21,093 is under the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,state_local_rule,False,"It applied SSI's $20/month unearned-income disregard instead of California's $230/month disregard, which replaces it. Using $230 yields $21,093 of countable income, under 138% FPL ($22,024.80)." -us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied the federal SSI $2,000 resource limit. California's Aged & Disabled resource limit for an individual is $130,000, so $4,200 passes easily." -us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (~$15,650) and gross income. The aged pathway allows up to 138% of the 2026 FPL ($22,024.80) after California's $230/month disregard, and countable income of $21,093 passes." -us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It held that the non-MAGI aged/disabled pathway requires disability, blindness, or SSI receipt. Age 65 or older alone meets the aged category, and the income ($21,093 after the disregard) and asset tests both pass." -us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,state_local_rule,False,"It compared retirement income to the aged Medi-Cal limit without California's $230/month disregard. That disregard reduces countable income to $21,093, which is under $22,024.80." -us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,state_local_rule,False,"It guessed that roughly $24k of income exceeds the limit without doing the calculation. Subtracting California's $2,760 annual disregard gives $21,093, which is under 138% of the 2026 FPL ($22,024.80)." -us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied the SSI $2,000 resource limit; California's Aged & Disabled limit is $130,000. It also skipped the $230/month income disregard, after which countable income of $21,093 meets the $22,024.80 limit." -us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the aged limit without California's $230/month disregard and wrongly required disability or SSI status. Age 67 alone meets the aged category, and income after the disregard ($21,093) is under 138% FPL." -us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It said the head fits no eligibility category, but age 67 meets the aged category on its own. It also skipped California's $230/month disregard, which brings countable income to $21,093, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,state_local_rule,False,"It subtracted the $1,165 of alimony paid, which SSI-based counting does not allow. It also applied only generic SSI-style disregards instead of California's $230 monthly disregard, which leaves countable income of $1,757.79 a month, under the 138% FPL limit of $1,835.40 a month. Its ~$1,800 monthly limit was also too low." +us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It subtracted SSI's $240-a-year general exclusion instead of California's $230 monthly ($2,760 annual) disregard, which replaces it. It also understated 138% FPL as ~$21,600 when the 2026 figure is $22,024.80. With the correct disregard, countable income is $21,093.47, which is under the limit." +us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used ~$18,564 for 138% FPL instead of $22,024.80 and compared gross income without California's $2,760 annual disregard. It also suggested $4,200 in bank assets could exceed the resource limit, but California's individual limit is $130,000." +us,scenario_031,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It assumed that being Medicare-age rules out Medicaid and never tested the aged non-MAGI pathway. Medicare eligibility bars only the 19–64 expansion group, and the head passes the senior-or-disabled income and asset tests." +us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied a 100% FPL limit (~$15,650) to the aged pathway, but the limit is 138% FPL ($22,024.80). It also skipped California's $230 monthly disregard, which brings countable income down to $21,093.47." +us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to a limit of $15,650–$21,600 without California's $2,760 annual disregard. It also understated the 2026 138% FPL limit, which is $22,024.80. Countable income of $21,093.47 is within the limit." +us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It tested gross income of $23,853 against an understated limit of ~$21,600 and never applied California's $230 monthly disregard. That disregard leaves $21,093.47 of countable income, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"It treated the MAGI expansion test as decisive and also considered a 100% FPL Aged & Disabled limit, using a 138% FPL figure of $20,783. It never applied California's $230 monthly disregard under the 138% FPL ($22,024.80) senior-or-disabled test, which countable income of $21,093.47 passes." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It wrongly excluded most of the Social Security income, which SSI rules count in full. It then tested against an SSI federal-benefit-rate limit (~$13,596) instead of the 138% FPL limit ($22,024.80). Applying the correct limit after California's $2,760 disregard to the full $23,853.47 gives eligibility." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"It claimed the head is not in a categorical group, but age 67 alone qualifies them as aged for the senior-or-disabled pathway. It also compared gross income to 138% FPL without California's $230 monthly disregard, which brings countable income under the limit." +us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It compared gross income of $23,853 directly to 138% FPL and omitted California's $2,760 annual disregard. That disregard reduces countable income to $21,093.47, below the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It used gross monthly income of $1,987.75 against an estimated limit of $1,732. After California's $230 monthly disregard, countable income is $1,757.79 a month, and the 2026 138% FPL limit is $1,835.40 a month." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It treated the lack of SSI receipt as relevant and tested gross income of $23,853. The aged pathway needs no SSI receipt, and California's $230 monthly disregard brings countable income to $21,093.47, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It compared gross income of $23,853 to a 138% FPL figure of $22,218 and skipped California's $2,760 annual disregard. The correct comparison is $21,093.47 of countable income against a $22,024.80 limit, which passes." +us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It compared gross income of $23,853 to ~$22,080 without applying California's $230 monthly disregard. The disregard lowers countable income to $21,093.47, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation, and its answer is consistent with testing gross income against an adult limit. The correct test is the aged pathway: countable income of $21,093.47 after California's $2,760 disregard, against a $22,024.80 limit (138% FPL), which passes." +us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It compared gross income of $23,853 to 138% FPL and never subtracted California's $230 monthly disregard. With the disregard, countable income is $21,093.47, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It tested gross monthly income against 138% FPL without California's $230 monthly disregard. That disregard leaves $1,757.79 a month, below the $1,835.40 limit." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no computation and concluded ineligibility. It missed that the 67-year-old head qualifies for the aged pathway, with countable income of $21,093.47 after California's $2,760 disregard, under the 138% FPL limit of $22,024.80." +us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the aged threshold and omitted California's $230 monthly disregard. The disregard reduces countable income to $21,093.47, within the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It asserted that income exceeds the non-MAGI limit without applying California's $230 monthly disregard. That disregard reduces countable income to $21,093.47, under the 138% FPL limit of $22,024.80." +us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It compared total income to 138% FPL without California's $2,760 annual disregard. Countable income after the disregard is $21,093.47, below the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed Medicare eligibility at 65+ rules out Medicaid, but in PolicyEngine Medicare bars only the adult expansion group. The aged senior-or-disabled pathway stays open, and the head passes its income and asset tests." +us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It applied the adult expansion group's MAGI test, but that group covers only ages 19–64 and the head is 67. It also overstated income at ~199% FPL. The applicable aged pathway uses SSI-based counting with California's $230 monthly disregard, giving $21,093.47 against a $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,age_disability,False,"It required disability or another qualifying flag, but age 67 alone meets the aged condition of the senior-or-disabled pathway. Countable income after California's $2,760 disregard is $21,093.47, within the 138% FPL limit." +us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,state_local_rule,False,"It evaluated a medically needy/spend-down standard instead of California's Aged & Disabled 138% FPL program. That program's $230 monthly disregard brings countable income to $21,093.47, under $22,024.80." +us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It declared income above the threshold without applying California's $230 monthly disregard. The disregard reduces countable income to $21,093.47, below the $22,024.80 senior-or-disabled limit." +us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It treated gross retirement income of $23,853 as countable and never subtracted California's $2,760 annual disregard. That leaves $21,093.47, under the aged pathway's 138% FPL limit of $22,024.80." +us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It tested income against an SSI-linked income level, but the senior-or-disabled pathway's limit is 138% FPL ($22,024.80). Countable income of $21,093.47, after California's $230 monthly disregard, passes that limit." +us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,state_local_rule,False,"It applied SSI's $20 general income exclusion, but California replaces that with a $230 monthly disregard. With the correct disregard, countable income is $21,093.47, below the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged threshold without California's $230 monthly disregard. That disregard brings countable income to $21,093.47, within the 138% FPL limit of $22,024.80." +us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It applied the SSI $20 general income exclusion instead of California's $230 monthly disregard, which replaces it. The correct disregard reduces countable income to $21,093.47, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It assumed PolicyEngine excludes Medicare-eligible people from Medicaid, but Medicare eligibility bars only the 19–64 expansion group. The aged senior-or-disabled pathway stays open, and the head qualifies through it." +us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It claimed $4,200 in bank assets exceeds the aged resource limit, but California's individual limit is $130,000. It also tested gross income of $23,853 without the $2,760 annual disregard, which brings countable income under $22,024.80." +us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,state_local_rule,False,"It treated $23,853 as countable income for the Aged & Disabled FPL test and omitted California's $230 monthly disregard. With the disregard, countable income is $21,093.47, under the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,state_local_rule,False,"It subtracted only the $20 monthly SSI exclusion, but California applies a $230 monthly disregard instead. That leaves $1,757.79 a month, under the $1,835.40 monthly limit (138% FPL)." +us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied a $2,000 asset limit, but California's 2026 senior-or-disabled resource limit is $130,000 for an individual. The $4,200 bank balance passes easily." +us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (~$15,650), but California's aged pathway allows income up to 138% FPL ($22,024.80). It also tested gross income without the $2,760 annual disregard, which brings countable income to $21,093.47." +us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,age_disability,False,"It ruled out the non-MAGI aged pathway because the head is not disabled, blind or an SSI recipient. Age 67 alone satisfies the aged condition, and the head passes the income test (after California's $230 monthly disregard) and the asset test." +us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,state_local_rule,False,"It judged retirement income to be above the aged limit without applying California's $230 monthly disregard. The disregard reduces countable income to $21,093.47, below the $22,024.80 limit." +us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,state_local_rule,False,"It guessed that income exceeds the limit without computing it. Correctly, the full $23,853.47 minus California's $2,760 disregard gives $21,093.47, under the $22,024.80 limit (138% FPL)." +us,scenario_031,head_medicaid_eligible,muse-wonder,llm_error,state_local_rule,False,"It subtracted only the $20 SSI disregard, getting ~$1,967 a month, but California uses a $230 monthly disregard. That disregard gives $1,757.79 a month, under the $1,835.40 limit." +us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied SSI's $2,000 resource limit, but California's senior-or-disabled limit is $130,000. It also compared gross income without the $230 monthly disregard, which brings countable income under 138% FPL." +us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"It treated Medicare eligibility and the absence of disability or SSI as barriers, but age alone qualifies the head as aged. It also tested gross income of $23,853 without California's $2,760 disregard, which leaves $21,093.47, under $22,024.80." +us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,age_disability,False,"It concluded the 67-year-old head lacks an eligibility category, but being 65 or older is itself the aged category of the senior-or-disabled pathway. Countable income after California's $230 monthly disregard is within 138% FPL." us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated gross income under 200% FPL as automatic BBCE eligibility and never ran the net income test PolicyEngine applies to this elderly household. Net income of about $1,779/mo ($1,987.75 minus the $209 standard deduction, with no shelter or medical deduction) is above the roughly $1,305/mo net limit. The household is ineligible, so the $23/mo minimum benefit ($276/yr) does not apply." us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model assumed BBCE at 200% FPL makes a 1-person household categorically eligible and entitled to the minimum benefit. It skipped the 100% FPL net income test that PolicyEngine applies to this elderly household. Net income of about $1,779/mo is above the roughly $1,305/mo limit, so the household is ineligible and gets $0, not $276." us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated countable income as very low. In fact, all $23,853 of Social Security, pension and IRA income counts as unearned income, which is $1,987.75/mo gross. Its $144/mo ($1,728/yr) benefit only works if net income is about $513/mo, which means it left out most of the pension and Social Security income. The real net income of about $1,779/mo fails the net income test, and 30% of it ($534) is more than the $298 maximum allotment, so SNAP is $0." @@ -2922,152 +2965,157 @@ us,scenario_032,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibilit us,scenario_032,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules expired for 2026, substituted a roughly $16,150 MFJ standard deduction and a $1,000 CTC, and thereby produced $1,477 of pre-credit tax minus $1,000. Under the applicable 2026 parameters, the $2,200 CTC supplies $500 of nonrefundable credit, which fully reduces the liability to $0; its remaining $1,700 and the EITC are refundable and do not enter this output." us,scenario_032,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits." us,scenario_032,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits." -us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The structure was right: full one-child EITC because AGI is below the MFJ phaseout, plus a $1,700 ACTC. But the model rounded the EITC down to ~$4,420 instead of using the published 2026 maximum of $4,427, which leaves the total $7 short." -us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted the 18-year-old spouse as a qualifying dependent and applied a 3-child EITC schedule, even though only the 6-year-old qualifies. It also used a $1,600 refundable CTC instead of $1,700 and applied an arbitrary phaseout. The correct amount is the full $4,427 one-child EITC plus a $1,700 ACTC." -us,scenario_032,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model claimed no refundable credit applies. In fact, an MFJ couple with $29,000 of wages and a 6-year-old qualifying child gets the full $4,427 one-child EITC and a $1,700 refundable ACTC." -us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025-level EITC of ~$4,328 instead of the 2026 maximum of $4,427, which AGI of $30,916 fully qualifies for. It then added an unexplained $215 'estimate refinement', which pushed the total above $6,127." -us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model placed $29,000 of earned income inside the one-child EITC phaseout and cut the credit to $3,372. The 2026 MFJ phaseout does not start until about $31,160, which is above both earnings and AGI of $30,916, so the full $4,427 applies." -us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model phased the EITC down to $3,412 even though income is below the MFJ phaseout start, so the full $4,427 applies. It then added a fabricated '$1,100 remaining' credit on top of the $1,700 ACTC." -us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $2,000 CTC with no $1,700 refundable cap, so its ACTC came out at $1,844. That figure came from a positive tax liability, which was wrong for two reasons: it used the outdated $30,000 standard deduction instead of $32,200, and it used the 85% tier for taxable Social Security instead of the 50% tier (taxable SS is $1,916). It also started the one-child EITC phaseout at ~$28,329 instead of ~$31,160, which cut the EITC to $4,329 instead of the full $4,427." -us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"After correctly reaching about $4,300 + $1,700, the model added an invented 'additional refundable credit for spouse as a qualifying dependent'. A spouse on a joint return is never a dependent and generates no refundable credit, so the answer should have been the $4,427 EITC plus the $1,700 ACTC." -us,scenario_032,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a nonexistent $5,177 one-child EITC maximum and an outdated $27,460 MFJ phaseout threshold. With the 2026 parameters ($4,427 maximum, phaseout starting around $31,160), AGI of $30,916 gets the unreduced $4,427, not $4,625." -us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the expired pre-TCJA ACTC rules ($1,000 cap, $3,000 earnings threshold). OBBBA set the 2026 CTC at $2,200 with a $1,700 refundable cap. It also overstated the one-child EITC as $4,488 instead of $4,427." -us,scenario_032,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the sunset-era $1,000 refundable CTC instead of the $1,700 OBBBA cap. It also phased the EITC down to $3,797 even though AGI is below the MFJ phaseout start, so the full $4,427 applies. Its stated total ($4,797) does not even match its submitted $5,430." -us,scenario_032,federal_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model started the one-child EITC phaseout at ~$23,052, which is roughly the single-filer threshold, and omitted the MFJ add-on, so the EITC was cut to $3,040. With the 2026 MFJ threshold of about $31,160, AGI of $30,916 gets the full $4,427; the $1,700 ACTC was correct." -us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model phased the EITC down to $3,599 even though AGI is below the MFJ phaseout threshold, so the full $4,427 applies. It also used the expired pre-TCJA $1,000 ACTC instead of the $1,700 2026 refundable cap." -us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The $5,462 total implies an EITC of about $4,462 plus the expired pre-TCJA $1,000 ACTC. Under OBBBA, the 2026 refundable CTC cap is $1,700 and the one-child EITC maximum is $4,427." -us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly used the 'pre-TCJA Additional Child Tax Credit of $1,000', assuming TCJA had sunset. OBBBA set the 2026 CTC at $2,200 with a $1,700 refundable portion. It also used $4,440 instead of the $4,427 EITC maximum." -us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used the expired $1,000 ACTC instead of the $1,700 2026 refundable cap. It also understated the EITC at $4,014, although AGI below the MFJ phaseout start qualifies for the full $4,427." -us,scenario_032,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"$3,745 is less than the EITC alone. The correct total is the full $4,427 one-child EITC (AGI of $30,916 is below the MFJ phaseout start) plus the $1,700 ACTC. The model's number reflects a phased-down EITC and a missing or understated refundable CTC." -us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and applied a $1,000 CTC with a $3,000 earnings threshold. OBBBA made the CTC $2,200 per child with a $1,700 refundable cap, so the ACTC is $1,700, not $1,000. Its EITC of $4,428 was essentially correct." -us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $5,248 total is consistent with an EITC of about $4,248 plus the expired pre-TCJA $1,000 ACTC. The correct components are the full $4,427 EITC and the $1,700 OBBBA refundable CTC cap." -us,scenario_032,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The $4,056 total is $2,071 short of the correct $4,427 EITC + $1,700 ACTC. It is consistent with a phased-down EITC paired with the expired $1,000 ACTC, even though AGI of $30,916 is below the MFJ phaseout start." -us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no usable value for federal_refundable_credits. The correct derivation is the $4,427 one-child EITC plus the $1,700 refundable ACTC, for $6,127." -us,scenario_032,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 CTC as refundable and ignored the $1,700 refundable (ACTC) cap. It also understated the one-child EITC maximum as $4,095 instead of $4,427." -us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable credit. A working MFJ couple with $29,000 of wages and a qualifying 6-year-old gets the $4,427 EITC and the $1,700 refundable ACTC." -us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed zero refundable credits because no credit amounts were listed in the inputs. Credits are computed, not given: the 6-year-old qualifies the couple for the $4,427 one-child EITC and the $1,700 ACTC." -us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The EITC of $4,427 was correct, but the model used an $1,800 refundable CTC limit instead of the 2026 $1,700 cap, which overstated the total by $100." -us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the right two credits but submitted $6,047, $80 below $4,427 + $1,700. The number is consistent with an EITC near $4,347 instead of the full 2026 one-child maximum of $4,427." -us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave a rough $2,500 guess with no computation. The full one-child EITC is $4,427 because AGI of $30,916 is below the MFJ phaseout start, and the ACTC is $1,700, so the guess misses about $3,600." -us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the sunset-era $1,000 CTC with a $3,000 earnings threshold instead of OBBBA's $2,200 CTC with a $1,700 refundable cap. It also projected the EITC at $4,436 instead of the published $4,427." -us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the 'post-sunset' $1,000 ACTC, but OBBBA kept the higher CTC, so the 2026 refundable portion is $1,700. It also understated the one-child EITC maximum as $4,318 instead of $4,427." -us,scenario_032,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The structure was correct: full EITC plus a $1,700 ACTC. But the model used a projected one-child EITC maximum of $4,436 instead of the 2026 parameter of $4,427, which overstated the total by $9." -us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model treated the CTC as $1,000 (pre-TCJA) instead of the 2026 $2,200 credit with a $1,700 refundable cap. It also overstated the EITC as $4,439 instead of $4,427." -us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly found the full EITC plus a $1,700 ACTC, but used an EITC maximum of $4,449 instead of the 2026 one-child maximum of $4,427, which overstated the total by $22." -us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable value for federal_refundable_credits. The correct derivation is the $4,427 one-child EITC plus the $1,700 refundable ACTC, for $6,127." -us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model applied a two-child EITC schedule even though only the 6-year-old is a qualifying child. It then left the refundable CTC out of the total entirely, reporting the EITC alone instead of $4,427 + $1,700." -us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model used a ~$22,190 phaseout threshold, roughly the single-filer level, and omitted the MFJ add-on, so the EITC was cut to $3,348. The 2026 MFJ threshold is about $31,160, above AGI of $30,916, so the full $4,427 applies; its $1,700 ACTC was correct." -us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model cycled through invented EITC maximums ($5,640, $5,386) and single-level phaseout thresholds before settling on $4,509. It then arbitrarily limited the ACTC to $800, even though 15% of $26,500 is $3,975 and well above the $1,700 cap. The correct amounts are a $4,427 EITC and a $1,700 ACTC." -us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model phased the EITC down to $1,171 and computed a $937 ACTC. AGI of $30,916 is below the MFJ phaseout start, so the full $4,427 applies, and 15% of earnings over $2,500 ($3,975) exceeds the $1,700 cap, so the ACTC is the full $1,700." -us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Compared $42,664 with 130% FPL (~$34,645) and ruled out SNAP/TANF categorical eligibility, but never applied Minnesota's universal free school meals program. That program makes the 6-year-old K-12 child eligible for free meals whatever the household earns." -us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Treated the 130% FPL income test and TANF/SNAP direct certification as the only free-meal pathways and overlooked Minnesota's universal free school meals program, which gives the K-12 child FREE tier status. Its ~170% FPL figure also overstates the engine's 1.56 ratio, but the missed state program is what decides the answer." -us,scenario_032,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,Applied only the federal 130% FPL free-meal limit to income at ~160% FPL and ignored that Minnesota provides universal free school meals. Under that program the household's K-12 child qualifies for free meals whatever the income. -us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Explicitly framed Minnesota eligibility as the NSLP 130% FPL test and concluded the household gets reduced-price meals at most. It missed that Minnesota's universal free school meals law makes every K-12 student eligible for free meals, so the income comparison is not decisive." -us,scenario_032,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Compared countable income of ~$42,700 with 130% FPL (~$34,600) and stopped there. It never accounted for Minnesota's universal free school meals program, which puts the K-12 child in the FREE tier." -us,scenario_032,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Correctly found income at ~157% FPL, above the 130% federal free-meal threshold, but ignored Minnesota's universal free school meals program. That program grants the K-12 child free meals without any income test." -us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied only the federal income test ($42,664 > 130% FPL of $35,326) and missed that Minnesota's universal free school meals program makes the household's K-12 child free-meal eligible regardless of income." -us,scenario_032,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Compared $42,664 with 130% FPL (~$35,750) and concluded ineligibility, never applying Minnesota's universal free school meals program. That program extends free meals to every K-12 student in the state." -us,scenario_032,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Explicitly limited its analysis to federal free school meals under the 130% FPL limit. It skipped Minnesota's universal free school meals program, which makes the 6-year-old eligible for free meals regardless of income." -us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Correctly summed $29,000 wages and $13,664 Social Security into $42,664 and found it above 130% FPL. It then treated that federal income test as final and missed Minnesota's universal free school meals program, which grants the K-12 child the FREE tier." -us,scenario_032,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,Used only the federal 130% poverty-guideline cutoff (income ~153% FPL) and never applied Minnesota's universal free school meals program. Under that program the K-12 child qualifies for free meals without an income test. -us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed free meals require SNAP/TANF categorical eligibility and defaulted to ineligible when those were absent. It ignored Minnesota's universal free school meals program, which makes the K-12 child eligible with no categorical or income condition." -us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Noted the school-age child but applied only the federal free-meal income limit and the categorical-status check. It missed that Minnesota's universal free school meals program places the child in the FREE tier regardless of the $42,664 income." -us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,Relied solely on the federal free-meal income threshold for a three-person household and never applied Minnesota's universal free school meals program. That program makes the K-12 child free-meal eligible whatever the household earns. -us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Checked only the federal income threshold and direct certification through benefit receipt. It omitted Minnesota's universal free school meals program, which is the decisive pathway giving the K-12 child free meals." -us,scenario_032,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Applied only the free-meal income threshold for a three-person household and missed Minnesota's universal free school meals program. That program grants free meals to the K-12 child without regard to income. -us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Correctly placed income at ~156% of the 2026 FPL, above the 130% federal free-meal limit, but treated that limit as dispositive. It ignored Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals." -us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"Found income at ~156% FPL and no SNAP/TANF categorical eligibility, then concluded PolicyEngine returns no free-meal support. It missed that PolicyEngine applies Minnesota's universal free school meals flag, which sets the child's tier to FREE regardless of income." -us,scenario_032,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Ruled out free meals through the 130% FPL income test and SNAP/TANF categorical eligibility, but never applied Minnesota's universal free school meals program. That program qualifies the K-12 child for free meals on its own." -us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Computed income at ~156.7% FPL and applied only the federal 130% free-meal threshold. It overlooked Minnesota's universal free school meals program, which grants the K-12 child free meals regardless of income." -us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"Compared gross income of ~$42,664 with 130% FPL ($33,566) and stopped at the federal income test. It missed Minnesota's universal free school meals program, which makes the household's K-12 child free-meal eligible." -us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for free_school_meals_eligible, so no answer was scored. The correct answer is Yes, because Minnesota's universal free school meals program puts the K-12 child in the FREE tier." -us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"Explicitly applied only PolicyEngine's federal school-meal income test (~156% of poverty, above 130%). It ignored the state_has_universal_free_school_meals pathway, which is true for Minnesota and sets the tier to FREE regardless of income." -us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Counted the $13,664 Social Security benefit, found income ~160% of poverty, and ruled out a SNAP/TANF categorical pathway. It missed the third pathway, Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals." +us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly identified the $4,427 one-child EITC maximum and full plateau eligibility, plus the $1,700 ACTC. It then rounded the EITC down to $4,420 and submitted $6,120 instead of $4,427 + $1,700 = $6,127." +us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It treated the 18-year-old spouse as a qualifying dependent and used a three-child EITC frame. Then it applied an invented phaseout to reach $3,995. The spouse is a joint filer, not a dependent, so only the 6-year-old qualifies, which gives the full one-child EITC of $4,427 plus the $1,700 ACTC." +us,scenario_032,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It denied all refundable credits, even though a joint return with $29,000 of earnings and a 6-year-old qualifying child is eligible for both the EITC and the ACTC. Those credits are the full $4,427 EITC and the $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It cited the $4,427 maximum but then used $4,328 (the 2025 maximum) for the EITC. It also added an arbitrary $215 'estimate refinement' that matches no credit. The correct total is $4,427 + $1,700 = $6,127." +us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It phased the one-child EITC down to $3,372, as if $29,000 were well into the phaseout range. Under the joint-filer threshold, AGI of $30,916 is still below the 2026 MFJ phaseout start, so the full $4,427 applies. Its $1,700 ACTC was correct." +us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It placed $29,000 in the phaseout range and cut the EITC to $3,412, when the full $4,427 applies below the MFJ phaseout start. It also added a nonexistent '$1,100 remaining' credit on top of the $1,700 ACTC." +us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed taxable Social Security with the $34,000 single-filer second threshold instead of the joint $32,000/$44,000 bases, and used a $30,000 standard deduction instead of $32,200, which created phantom tax of $156. It then set the ACTC at $1,844 without the $1,700 per-child refundable cap. It also phased the EITC to $4,329 from an underestimated MFJ phaseout start of $28,329, when the full $4,427 applies." +us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It reached about $6,000 from the EITC plus the $1,700 ACTC. It then inflated the total to $8,200 by adding an invented refundable credit for the spouse as a 'qualifying dependent'. A joint-filing spouse generates no refundable credit, and the EITC is $4,427, which gives $6,127." +us,scenario_032,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a wrong one-child EITC maximum of $5,177 (the actual 2026 maximum is $4,427) and a $27,460 MFJ phaseout threshold, ending with an EITC of $4,625. AGI of $30,916 sits below the actual 2026 MFJ phaseout start, so the EITC is exactly $4,427." +us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA ACTC rules ($1,000 cap, 15% of earnings over $3,000) instead of the OBBBA rules for 2026, under which the refundable CTC is $1,700. It also overstated the EITC maximum as $4,488 instead of $4,427." +us,scenario_032,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It phased the EITC down to $3,797 even though AGI of $30,916 is below the MFJ phaseout start, where the full $4,427 applies. It also used the pre-TCJA $1,000 refundable CTC instead of the $1,700 OBBBA cap. Its submitted $5,430 does not even match its own stated $4,797 total." +us,scenario_032,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It started the EITC phaseout at about $23,052, which is the single-filer threshold, without the joint-filer increase, and cut the EITC to $3,040. With the MFJ threshold, AGI of $30,916 is still on the plateau, so the EITC is the full $4,427. Its $1,700 ACTC was correct." +us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It phased the EITC down to $3,599.34, when the full $4,427 applies below the 2026 MFJ phaseout start. It also used the sunset-era $1,000 refundable CTC instead of the $1,700 cap that OBBBA set for 2026." +us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $5,462 matches an EITC of about $4,462 plus a $1,000 pre-TCJA refundable CTC. Under 2026 law the refundable CTC is $1,700 and the one-child EITC is $4,427, for a total of $6,127." +us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly applied the pre-TCJA $1,000 Additional Child Tax Credit, ignoring OBBBA's permanent extension with a $1,700 refundable cap for 2026. It also overstated the EITC maximum as $4,440 instead of $4,427." +us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It understated the EITC at $4,014 instead of the full $4,427, which applies because AGI is below the MFJ phaseout start. It also used a $1,000 ACTC instead of the $1,700 refundable cap in effect for 2026." +us,scenario_032,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $3,745 is below even the full $4,427 one-child EITC on its own. That is consistent with applying an EITC phaseout that does not reach a joint return at $30,916 AGI, and with dropping or shrinking the $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed TCJA had expired for 2026 and used a $1,000 CTC with a refundable portion of 15% over $3,000. OBBBA extended the CTC with a $1,700 refundable cap, so the ACTC is $1,700. Its EITC of $4,428 was off by $1 from $4,427." +us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $5,248 matches an EITC of about $4,248 plus a $1,000 sunset-era ACTC. The correct components are the full $4,427 EITC and the $1,700 refundable CTC under 2026 law." +us,scenario_032,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $4,056 is far below the correct $4,427 EITC plus $1,700 ACTC. That is consistent with phasing out an EITC that is not in its phaseout range at $30,916 joint AGI, and with using the $1,000 pre-TCJA ACTC." +us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_refundable_credits, so there was nothing to score against the $6,127 reference." +us,scenario_032,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It treated the entire $2,200 CTC as refundable. The refundable portion is capped at $1,700 per child for 2026. It also understated the plateau EITC as $4,095 instead of the $4,427 maximum." +us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It identified no refundable credit, missing that $29,000 of earned income with a 6-year-old qualifying child on a joint return gives the full $4,427 EITC and a $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed zero refundable credits because none were 'specified'. The EITC and ACTC are computed from earnings and the qualifying child: $4,427 plus $1,700." +us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got the full $4,427 EITC right but used a refundable CTC cap of $1,800. The 2026 refundable CTC limit is $1,700 per child, so the total is $6,127, not $6,227." +us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It identified the right two credits but submitted $6,047, which implies an EITC of about $4,347 alongside the $1,700 ACTC. That is consistent with an inflation-adjusted 2025 maximum rather than the published 2026 one-child maximum of $4,427." +us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave an unsupported round $2,500 with no computation. The correct derivation is the full one-child EITC of $4,427 (joint AGI below the phaseout start) plus the $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the post-sunset $1,000 CTC with a refundable portion of 15% over $3,000, instead of OBBBA's $1,700 refundable cap. It also projected the EITC maximum at $4,436 instead of the actual 2026 figure of $4,427." +us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the post-sunset $1,000 additional CTC instead of the $1,700 refundable cap that OBBBA set for 2026. It also understated the one-child EITC maximum as $4,318 instead of $4,427." +us,scenario_032,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its structure was correct: full EITC plus the $1,700 ACTC. However, it projected the 2026 one-child EITC maximum as $4,436 instead of the published $4,427, which overshot by $9." +us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $1,000 CTC as the refundable amount, ignoring OBBBA's $2,200 CTC with its $1,700 refundable cap for 2026. It also overstated the EITC maximum as $4,439 instead of $4,427." +us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It correctly applied the full EITC and the $1,700 ACTC cap but overestimated the 2026 one-child EITC maximum as $4,449 instead of $4,427." +us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_refundable_credits, so there was nothing to score against the $6,127 reference." +us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"It used the two-child EITC schedule, although the household has only one qualifying child, the 6-year-old. It also dropped the refundable CTC from its total despite computing it. The correct total is the one-child EITC of $4,427 plus the $1,700 ACTC." +us,scenario_032,federal_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It applied an EITC phaseout at $30,916 AGI and reduced the credit to $4,257. That AGI is below the 2026 MFJ phaseout start, so the full $4,427 applies. Its $1,700 ACTC was correct." +us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It started the EITC phaseout at about $22,190, roughly the single-filer threshold, without the joint-filer increase, and cut the EITC to $3,348. Under the MFJ threshold, the full $4,427 applies. Its $1,700 ACTC was correct." +us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used inflated EITC maxima ($5,386 to $5,640) with a phaseout starting near $22,000 to $23,500 and arrived at $4,509. It then dropped the refundable CTC to an unsupported $800, after its own work had shown the $1,700 cap applies. The correct total is $4,427 + $1,700." +us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It drastically understated both credits, with an EITC of $1,171 and an ACTC of $937. The correct amounts are the full one-child EITC of $4,427 (joint AGI below the phaseout start) and the full $1,700 ACTC, since 15% of earnings over $2,500 equals $3,975, which is above the cap." +us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model compared $42,664 to the federal 130% FPL free-meal limit and checked for SNAP/TANF categorical eligibility, then stopped. It never applied Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals at any income." +us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model ruled out free meals using the federal 130% FPL threshold and the lack of direct certification. It missed that Minnesota provides free school meals to every K-12 student, so the household gets FREE-tier support even at 1.56x FPL." +us,scenario_032,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model used only the federal 130% FPL free-meal limit. It ignored Minnesota's universal free school meals program, which gives the 6-year-old K-12 student free meals regardless of the household being at about 156% FPL." +us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said that free meals in Minnesota require income at or below 130% FPL. That is wrong: Minnesota has universal free school meals for all K-12 students, so the income test does not apply and the child qualifies for the FREE tier." +us,scenario_032,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model denied free meals because income was above 130% FPL for a family of 3. It did not account for Minnesota's universal free school meals program, which gives every K-12 student free meals with no income test." +us,scenario_032,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,The model applied only the federal 130% FPL free-meal threshold to income at about 157% FPL. It missed that Minnesota's universal free school meals program makes the K-12 child eligible regardless of income. +us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model's only test was $42,664 against 130% FPL. It ignored Minnesota's universal free school meals law, which puts every Minnesota K-12 student in the FREE tier." +us,scenario_032,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model ruled out free meals because income exceeded 130% FPL for 3. It skipped Minnesota's universal free school meals program, under which the enrolled 6-year-old gets free meals regardless of income." +us,scenario_032,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,The model applied the federal 130% FPL free-meal test and stopped there. Minnesota's universal free school meals program overrides the federal income test and gives the K-12 child free meals. +us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model correctly totaled $42,664 in income and compared it to 130% FPL, but it treated that federal income test as the only route to free meals. Minnesota's universal free school meals program makes every K-12 student eligible." +us,scenario_032,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model ruled out free meals because income was about 153% FPL, above the 130% cutoff. It missed Minnesota's universal free school meals program, which gives free meals to all K-12 students with no income test." +us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model said no because there was no SNAP/TANF categorical eligibility. It overlooked that Minnesota provides universal free school meals, so the K-12 child qualifies without categorical eligibility or an income test." +us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,The model found the school-age child but then applied only the federal free-meal income limit and the categorical test. It did not recognize that Minnesota's universal free school meals program gives that child free meals at any income. +us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied only the federal free-meal income threshold. It missed that Minnesota is a universal free school meals state, which puts the household's K-12 student in the FREE tier regardless of income." +us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model relied on the federal income threshold and direct certification and found neither met. It missed Minnesota's universal free school meals program, which makes the K-12 child eligible independent of both tests." +us,scenario_032,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model applied only the federal free-meal income threshold for a three-person household. It ignored Minnesota's universal free school meals program, which gives every K-12 student free meals." +us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model correctly put income at about 156% FPL but treated the federal 130% limit as decisive. Minnesota's universal free school meals program makes the K-12 child eligible regardless of that ratio. +us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model checked only the 130% FPL limit and SNAP/TANF categorical eligibility. It missed the third route PolicyEngine uses: Minnesota's universal free school meals flag, which sets the tier to FREE." +us,scenario_032,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model ruled out the federal income test and the categorical route through zero SNAP/TANF. It never considered Minnesota's universal free school meals program, which gives the K-12 child free meals regardless of income or benefit receipt." +us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model compared income to the 130% FPL federal free-meal threshold and stopped. It missed that Minnesota provides universal free school meals, so the household's K-12 student qualifies for the FREE tier." +us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL test. It did not apply Minnesota's universal free school meals program, which gives the enrolled 6-year-old free meals regardless of household income." +us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for free_school_meals_eligible. The correct answer is 1, because Minnesota's universal free school meals program puts the household's K-12 child in the FREE tier." +us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model claimed to use PolicyEngine's federal income test, but PolicyEngine also checks state_has_universal_free_school_meals, which is True for Minnesota. That check puts the child in the FREE tier despite income at 156% FPL." +us,scenario_032,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model ruled out free meals because income exceeded the 130% of poverty limit. It missed Minnesota's universal free school meals program, which removes the income test for all K-12 students." +us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model ruled out both the 130% FPL income route and the SNAP/TANF categorical route. It did not consider Minnesota's universal free school meals program, which gives the K-12 child free meals regardless of either." us,scenario_032,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It excluded the $13,664 in Social Security dependent benefits as non-taxable and tested only about $29,000 against 138% FPL. Medicaid MAGI adds back all non-taxable Social Security, so countable income is $42,664, or 156% FPL, which is above the 138% limit." -us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly counted the Social Security benefits and found MAGI of about $42,664, which is above 138% FPL. It then claimed eligibility through a higher Medicaid limit for parents and caretakers, citing MinnesotaCare. MinnesotaCare is a Basic Health Program, not Medicaid, and Minnesota's Medical Assistance limit for parents is 133% FPL plus the 5% disregard, which the Head's 156% FPL exceeds." -us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as excluded from MAGI and counted only the $29,000 in wages. Medicaid MAGI includes the non-taxable part of Social Security, which raises income to $42,664, or 156% FPL, above the 138% expansion-adult limit." -us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly found that $42,664 exceeds 138% FPL, then invented a 275% FPL Medicaid limit for parents and caretakers in Minnesota. Minnesota's Medical Assistance limit for parents is 133% FPL plus the 5% disregard, so at 156% FPL the Head has no Medicaid pathway." -us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It counted the Social Security benefit as a fourth household member, which made the household size 4 and the limit about $43,000. The Medicaid household is 3 people (Head, Spouse, Child 1), so the $42,664 MAGI is 156% of the 3-person FPL and fails the 138% limit." -us,scenario_032,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its MAGI of $30,916 is wages plus only the taxable part of Social Security (about $1,916). Medicaid MAGI adds back the non-taxable Social Security as well, giving $42,664, or 156% FPL for 3 people, which exceeds the 138% limit." -us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted low income without computing MAGI. That is consistent with leaving out the $13,664 in Social Security benefits, which Medicaid MAGI counts in full. The correct MAGI is $42,664, or 156% FPL for a 3-person household, above Minnesota's 138% adult limit." -us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It placed household MAGI within the 138% FPL expansion range, which means it left out the non-taxable Social Security dependent benefits. With the full $13,664 added back, as Medicaid MAGI requires, income is $42,664, or 156% FPL, and the Head is ineligible." -us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,taxable_income_or_deductions,False,"It labeled the household low-income without computing MAGI, which implies it counted only wages. Medicaid MAGI includes all $13,664 in Social Security benefits, which brings income to $42,664, or 156% FPL, above Minnesota's 138% expansion-adult limit." -us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a MAGI of $30,916, meaning wages plus only the taxable part of Social Security, and got 113% FPL. Medicaid MAGI adds back the non-taxable Social Security, giving $42,664, or 156% FPL, which exceeds the 138% limit." -us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its 113.6% FPL figure comes from counting only wages plus the taxable part of Social Security. Medicaid MAGI includes the full $13,664 in Social Security benefits, which puts the Head at 156% FPL, above Minnesota's 138% limit for adults aged 19 to 64." -us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for head_medicaid_eligible, so this output is missing rather than a substantive Medicaid determination." -us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It asserted income at 138% of poverty without computing MAGI and treated that as qualifying. With wages of $29,000 plus the full $13,664 in Social Security, which Medicaid MAGI counts, the Head's MAGI is 156% FPL, above the 138% expansion limit." -us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its MAGI range of $29,000 to $30,916 counts wages plus at most the taxable part of Social Security. Medicaid MAGI adds back all non-taxable Social Security, which gives $42,664, or 156% of the 3-person FPL, above the 138% adult limit." -us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It computed $42,664 in MAGI, then dropped the Social Security dependent benefits on the theory that a child receives them. It tested only the $29,000 in wages. The benefits are the Head's own income, and Medicaid MAGI counts them in full, so the Head is at 156% FPL, above the 138% limit." +us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model claimed Social Security benefits are excluded from MAGI and counted only the ~$29,000 in wages. Medicaid MAGI adds back the non-taxable portion of Social Security, so the full $13,664 counts. That makes MAGI $42,664, about 156% FPL, which is above the 138% limit." +us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly counted $42,664 including Social Security and saw that this is above 138% FPL. It then made the head eligible through a higher Minnesota threshold for parents and caretakers, which does not exist. Minnesota Medical Assistance covers parents at the same 133%+5% (138%) FPL limit, and MinnesotaCare is a Basic Health Program, not Medicaid." +us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated Social Security dependent benefits as generally excluded from MAGI and counted only the $29,000 in wages. MAGI includes non-taxable Social Security, so countable income is $42,664, about 156% FPL, which is above the 138% expansion-adult limit." +us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly found that $42,664 is above 138% FPL for a household of 3. It then applied a Minnesota Medicaid limit of 275% FPL for parents, which does not exist. Minnesota Medical Assistance for parents and caretakers uses the 138% FPL limit, so at 156% FPL the head is ineligible." +us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used a household size of 4 by treating the Social Security dependent benefit as if it added a household member. That raised the 138% FPL threshold to about $43,000. The household has 3 people, and 138% of the 3-person FPL is about $37,700, which is below the $42,664 MAGI." +us,scenario_032,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model's MAGI of $30,916 is wages plus only the taxable portion of Social Security (about $1,916). Medicaid MAGI adds back the non-taxable Social Security too, giving $42,664, which is about 156% FPL and above 138%." +us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model asserted low income without computing MAGI. Counting the full $13,664 in Social Security dependent benefits plus $29,000 in wages gives MAGI of $42,664, which is 156% FPL for 3 people and above Minnesota's 138% limit for adults." +us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model asserted MAGI was within the 138% expansion range, which only holds if the non-taxable Social Security benefits are left out. Medicaid MAGI includes all $13,664 of those benefits, which puts household MAGI at $42,664, or 156% FPL, and makes the head ineligible." +us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model labeled the household low-income without computing MAGI. Wages of $29,000 plus all $13,664 in Social Security benefits (non-taxable Social Security is added back in MAGI) give $42,664, which is 156% FPL and above Minnesota's 138% adult limit." +us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $30,916 (wages plus only taxable Social Security), which works out to about 113% FPL. Medicaid MAGI adds back the non-taxable Social Security, so MAGI is $42,664, which is 156% FPL and above the 138% limit." +us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's 113.6% FPL figure counts only wages plus the taxable portion of Social Security. Adding back the non-taxable Social Security dependent benefits as MAGI requires gives $42,664, which is about 156% FPL and above Minnesota's 138% adult limit." +us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no answer or reasoning for head_medicaid_eligible. The correct result is 0, because MAGI of $42,664 including all Social Security benefits is 156% FPL, above the 138% limit, and no other Medicaid category applies." +us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted household income was at 138% of poverty and treated that as qualifying. Wages plus all $13,664 in Social Security benefits give MAGI of $42,664, which is 156% FPL and clearly above the 138% expansion limit." +us,scenario_032,head_medicaid_eligible,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model based MAGI only on the $29,000 in wages and left out the $13,664 in Social Security dependent benefits. MAGI includes non-taxable Social Security, so MAGI is $42,664, which is 156% FPL and above 138%." +us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model put MAGI at $29,000 to $30,916, counting at most the taxable portion of Social Security. Medicaid MAGI adds back all non-taxable Social Security, so MAGI is $42,664, which is 156% FPL and above the 138% adult limit." +us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model first counted $42,664 and then dropped the Social Security benefits on the theory that a child's benefits are excluded. The benefits are listed as the head's income, so they count in full in the head's MAGI. That puts income at 156% FPL, above the 138% limit." us,scenario_032,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a household-level “WIC-eligible context” pathway and transferred eligibility to the head based on living with a child. The head is age 50, is not pregnant, postpartum, or breastfeeding, and therefore fails WIC's person-specific categorical requirement; moreover, the only child is age 6 and is not categorically eligible either." -us,scenario_032,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly ($1,798 + $420.50) but said Minnesota has no employee payroll tax in 2026, so it left out the $127.60 Paid Leave contribution (0.44% of $29,000). Its $2,218.50 fits the reading where the employer pays the whole 0.88% premium. The reference instead assumes the employer withholds the full 0.44% employee share." -us,scenario_032,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $29,000 and treated Minnesota as having no mandatory employee payroll tax, so it left out the 0.44% Paid Leave employee share ($127.60). That fits the reading where the employer pays the full premium, not the reference's reading where the employer withholds the maximum employee share." -us,scenario_032,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It derived $2,218.50 in FICA several times, then reported $2,175.90 after an unexplained 'state-specific adjustment' that matches no rule. It also never added the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000). The correct total is $2,346.10." -us,scenario_032,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It rounded 7.65% FICA on $29,000 to $2,219 and said Minnesota has no mandatory employee payroll tax, so it left out the $127.60 Paid Leave contribution (0.44% employee share). That follows the reading where the employer pays the full premium. The reference assumes the employee share is withheld." -us,scenario_032,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA on $29,000 and said Minnesota has no mandatory employee payroll tax, which drops the 0.44% Paid Leave employee contribution ($127.60). Its $2,218.50 matches the employer-pays-all reading, not the reference's maximum-withholding reading." -us,scenario_032,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It computed Social Security ($1,798) and Medicare ($420.50) correctly but said Minnesota has no mandatory employee payroll tax, so it left out the $127.60 Paid Leave employee share (0.44% of wages). That is the employer-pays-full-premium reading. The reference assumes the employer withholds the employee half." -us,scenario_032,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It counted only employee FICA ($2,218.50) and said Minnesota has no mandatory employee payroll tax, leaving out the 0.44% Paid Leave contribution of $127.60. That fits the reading where the employer pays the whole premium, not the reference's assumption that the employee share is withheld." -us,scenario_032,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It applied only the 7.65% FICA rate to $29,000 and never considered the Minnesota Paid Leave employee contribution (0.44%, $127.60). Its total matches the reading where the employer pays the full premium, not the reference's maximum-withholding reading." -us,scenario_032,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no employee-side payroll tax beyond federal FICA, missing that Minnesota Paid Leave (effective 2026) lets the employer withhold a 0.44% employee share, which is $127.60 here. Its $2,218.50 matches the employer-pays-all reading. The reference assumes the share is withheld." -us,scenario_032,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no paid family leave employee contribution in 2026. In fact Minnesota Paid Leave began January 1, 2026, and the engine withholds a 0.44% employee share ($127.60). Its FICA-only $2,218.50 fits the reading where the employer pays the whole premium." -us,scenario_032,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It assumed no Minnesota employee payroll tax is modeled, so it left out the 0.44% Paid Leave contribution on $29,000 ($127.60). That is the employer-pays-full-premium reading, not the reference's maximum-employee-withholding reading." -us,scenario_032,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA ($2,218.50) and said no employee state payroll tax applies, so it left out the $127.60 Minnesota Paid Leave employee share (0.44%). That follows the reading where the employer pays the full premium, not the reference's withholding assumption." -us,scenario_032,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It correctly included Minnesota Paid Leave but used a 0.35% employee rate ($101.50) instead of 0.44%, which is half of the 0.88% total premium ($127.60). That understated the total by $26.10: it answered $2,320 instead of $2,346.10." -us,scenario_032,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Its submitted $2,320 contradicts its own explanation, which says no state payroll tax and gives $2,218.50. The $2,320 equals FICA plus a 0.35% Paid Leave rate, not the correct 0.44% employee share ($127.60), which gives $2,346.10." -us,scenario_032,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It applied only 7.65% FICA to $29,000 and left out the Minnesota Paid Leave employee contribution (0.44%, $127.60). That fits the reading where the employer pays the full premium. The reference assumes the maximum employee share is withheld." -us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It computed employee FICA only, rounded to $2,219, and never added the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000). That follows the employer-pays-all reading, not the reference's withholding reading." -us,scenario_032,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It calculated 29,000 × 7.65% = $2,218.50 and left out the 0.44% Minnesota Paid Leave employee share ($127.60). Its answer fits the reading where the employer pays the whole premium, not the reference's assumption of maximum employee withholding." -us,scenario_032,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted only the 7.65% FICA share and left out the Minnesota Paid Leave employee contribution of 0.44% on $29,000 ($127.60). That matches the employer-pays-full-premium reading. The reference assumes the employee share is withheld." -us,scenario_032,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It reported FICA only ($2,218.50) and left out the $127.60 Minnesota Paid Leave employee contribution (0.44%). That fits the reading where the employer pays the full premium, not the reference's maximum-withholding reading." -us,scenario_032,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 and did not add the 0.44% Minnesota Paid Leave employee contribution ($127.60). That is the employer-pays-all reading, not the reference's assumption that the employer withholds the maximum employee share." -us,scenario_032,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It computed employee FICA of $2,218.50 and left out the $127.60 Minnesota Paid Leave employee share (0.44% of wages). That follows the reading where the employer pays the full 0.88% premium." -us,scenario_032,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It summed Social Security ($1,798) and Medicare ($420.50) but left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). That matches the employer-pays-all reading, not the reference's withholding reading." -us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation. The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave employee share = $2,346.10." -us,scenario_032,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $29,000 and left out the Minnesota Paid Leave employee contribution (0.44%, $127.60). That follows the reading where the employer pays the full premium, not the reference's maximum-withholding reading." -us,scenario_032,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Its explanation describes only the employee 6.2% + 1.45% rates, but it submitted $4,428.60. That is roughly double the $2,218.50 employee FICA, which suggests it added the employer FICA match that the question excludes. It also left out the $127.60 Minnesota Paid Leave employee share; the correct total is $2,346.10." -us,scenario_032,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It correctly computed 7.65% × $29,000 = $2,218.50, then dropped that for an invented 5.24% rate tied to a nonexistent wage-base interaction ($1,520). $29,000 is far below the Social Security wage base, so the full 6.2% applies. It also never added the $127.60 Minnesota Paid Leave contribution." -us,scenario_032,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It computed FICA of $2,218.50 and said no state payroll tax applies, leaving out the 0.44% Minnesota Paid Leave employee share ($127.60). That is the employer-pays-full-premium reading, not the reference's withholding reading." -us,scenario_032,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA (rounded to $2,219) and left out the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000). That fits the reading where the employer pays the whole premium." -us,scenario_032,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It counted only 7.65% FICA and said no other employee payroll tax was indicated, which drops the 0.44% Minnesota Paid Leave contribution ($127.60). That matches the employer-pays-all reading, not the reference's maximum-withholding reading." -us,scenario_032,payroll_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It reported 7.65% of $29,000 and left out the Minnesota Paid Leave employee share (0.44%, $127.60). That follows the reading where the employer pays the full premium, not the reference's withholding assumption." -us,scenario_032,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It computed only Social Security and Medicare on $29,000 and left out the $127.60 Minnesota Paid Leave employee contribution (0.44%). That fits the reading where the employer pays the whole premium." -us,scenario_032,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It said no mandatory Minnesota employee payroll tax applies, which drops the 0.44% Paid Leave employee share ($127.60) the engine withholds. Its $2,218.50 matches the employer-pays-full-premium reading." -us,scenario_032,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no employee-side payroll tax and so left out the 0.44% Paid Leave contribution on $29,000 ($127.60). That is the employer-pays-all reading. The reference assumes the employer withholds the maximum employee share." -us,scenario_032,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly saw that Minnesota Paid Leave employee withholding is optional for the employer and excluded it, giving $2,218.50. The reference takes the other reading: the employer withholds the maximum 0.44% employee share ($127.60), for $2,346.10." -us,scenario_032,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It summed $1,798 Social Security and $420.50 Medicare and never considered the 0.44% Minnesota Paid Leave employee contribution ($127.60). That fits the reading where the employer pays the full premium." -us,scenario_032,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It said no state employee payroll taxes apply, leaving out the $127.60 Minnesota Paid Leave employee share (0.44% of $29,000). That follows the employer-pays-all reading, not the reference's maximum-withholding reading." -us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation. The correct total is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave employee contribution = $2,346.10." -us,scenario_032,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It rounded employee FICA to $2,219 and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). That matches the reading where the employer pays the full premium, not the reference's withholding reading." -us,scenario_032,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It said Minnesota levies no mandatory employee payroll tax, which drops the 0.44% Paid Leave employee share ($127.60) that started in 2026. Its $2,218.50 fits the employer-pays-full-premium reading." -us,scenario_032,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It dismissed Minnesota payroll taxes because Minnesota has no disability insurance tax like CA or NY, overlooking the 2026 Minnesota Paid Leave employee share (0.44%, $127.60). Its rounded $2,219 follows the reading where the employer pays the full premium." -us,scenario_032,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It reported 7.65% FICA ($2,218.50, though it misstated Medicare as $419.50) and left out the $127.60 Minnesota Paid Leave employee contribution (0.44%). That matches the employer-pays-all reading, not the reference's maximum-withholding reading." -us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model correctly placed $42,664 between 130% and 185% FPL for a 3-person household. It then applied the federal reduced-price band alone and ignored Minnesota's universal free school meals program. That program puts every student in the FREE tier, which replaces reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model counted a 4-person household with two children, but the household has 3 people and one child. Its answer rests on the federal 185% FPL reduced-price limit alone. It missed that Minnesota's universal free school meals program gives the child the FREE tier, so reduced-price support is not returned." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model concluded that income above 130% FPL rules out free meals, which leaves only reduced-price. That reasoning ignores Minnesota's universal free school meals program, which gives free meals to every student regardless of income. The household is therefore in the FREE tier, not the reduced-price tier." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model placed the 3-person household's $42,664 income in the federal 130-185% reduced-price band and stopped. It never applied Minnesota's universal free school meals program, which puts the 6-year-old in the FREE tier and so replaces reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model used only the federal 130-185% FPL band, putting the household at about 170% FPL (the engine ratio is 1.56). It ignored Minnesota's universal free school meals program, which assigns the FREE tier to every student and so makes reduced-price eligibility false." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model read income of about 160% FPL as reduced-price eligible under the federal 130-185% band. It missed that Minnesota provides universal free school meals, which puts the household in the FREE tier instead of reduced-price." -us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model worked repeatedly through the federal 130%/185% FPL thresholds for a family of 3. It concluded that income above 130% FPL rules out free meals. It never considered Minnesota's universal free school meals program, which gives the child free meals regardless of income and so removes reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model applied only the federal rule that income between 130% and 185% FPL means reduced-price. It ignored Minnesota's universal free school meals program, under which the student gets the FREE tier and reduced-price support is not returned." -us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model correctly put income at about 157% FPL but treated the federal 130-185% band as decisive. It missed Minnesota's universal free school meals program, which gives the child the FREE tier and so replaces reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model's only test was income below 185% FPL. It ignored Minnesota's universal free school meals program, which places every student in the FREE tier and so makes reduced-price eligibility false." -us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model placed income between 130% and 185% FPL for 3 and concluded the household was reduced-price eligible. It never applied Minnesota's universal free school meals program, which puts the household in the FREE tier instead." -us,scenario_032,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model used the federal 130-185% FPL band for a household of 3. It missed that Minnesota's universal free school meals program gives Child 1 free meals regardless of income, which replaces reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model placed $42,664 between 130% and 185% of the 2026 guideline and concluded reduced-price. It ignored Minnesota's universal free school meals program, which assigns the FREE tier to every student." -us,scenario_032,reduced_price_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model correctly placed income at about 153% of the guideline, inside the 130-185% band. It missed that Minnesota's universal free school meals program puts the household in the FREE tier, so no reduced-price support is returned." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model found income above the federal free-meal limit and below the reduced-price limit, and concluded reduced-price. That treats the federal income limits as final and ignores Minnesota's universal free school meals program, which gives every student free meals regardless of income." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model checked income against the federal reduced-price threshold and nothing else. It never applied Minnesota's universal free school meals program, which gives the household the FREE tier in place of the reduced-price tier." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model placed household income in the reduced-price income range for 3 people. It ignored Minnesota's universal free school meals program, which gives the FREE tier to every student and so makes reduced-price eligibility false." -us,scenario_032,reduced_price_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model based its answer only on income falling in the reduced-price range for 3 people. It missed that Minnesota's universal free school meals program moves the student into the FREE tier, which replaces reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model correctly put income at about 156% FPL and applied the 185% reduced-price limit. It ignored Minnesota's universal free school meals program, which gives the household free meals and so no reduced-price support." -us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model placed income at about 156% FPL, between 130% and 185%, and concluded reduced-price. It did not apply Minnesota's universal free school meals program, under which PolicyEngine assigns the FREE tier regardless of income." -us,scenario_032,reduced_price_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model found gross income between 130% and 185% FPL with a school-age child and concluded reduced-price. It missed Minnesota's universal free school meals program, which puts Child 1 in the FREE tier and so replaces reduced-price eligibility." -us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model put income at about 156.7% of the 2026 FPL, below the 185% reduced-price threshold, and stopped there. It ignored Minnesota's universal free school meals program, which gives the student free meals regardless of income." -us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model placed income between its computed 130% and 185% FPL dollar limits ($33,566 to $47,767) and concluded reduced-price. It never applied Minnesota's universal free school meals program, which puts the household in the FREE tier." -us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for reduced_price_school_meals_eligible, so its response has no answer to score. The correct answer is 0: Minnesota's universal free school meals program puts the household in the FREE tier." -us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model placed income at about 156% of poverty, between the 130% free-meal limit and the 185% reduced-price limit, and treated those federal limits as final. It missed that Minnesota's universal free school meals program gives every student free meals regardless of income." -us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model applied only the federal 185% reduced-price cutoff to income of about 160% of poverty. It ignored Minnesota's universal free school meals program, which gives the school-age child the FREE tier and so makes reduced-price eligibility false." +us,scenario_032,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It computed FICA of $2,218.50 correctly but said Minnesota has no mandatory employee state payroll tax in 2026. That missed the Minnesota Paid Leave premium that began in 2026. PolicyEngine treats the 0.44% employee share ($127.60) as withheld, and the prompt does not say whether the employer pays it instead." +us,scenario_032,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $29,000 and said there was no modeled mandatory Minnesota employee payroll tax. That left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60. PolicyEngine includes this amount by assuming the employer withholds the maximum employee share." +us,scenario_032,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It derived $2,218.50 in FICA several times, then cut the figure to $2,175.90 with no basis, citing unspecified 'state-specific considerations'. It also left out the Minnesota Paid Leave employee contribution (0.44% × $29,000 = $127.60), which raises the total to $2,346.10, not lowers it." +us,scenario_032,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It rounded 7.65% FICA to $2,219 and said Minnesota has no mandatory employee payroll tax. That left out the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60. PolicyEngine counts this amount, assuming maximum employer withholding." +us,scenario_032,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA of $2,218.50 and said Minnesota has no mandatory employee payroll tax. That left out the Minnesota Paid Leave contribution (0.44% of $29,000 = $127.60), which PolicyEngine adds as employee state payroll tax." +us,scenario_032,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Its Social Security ($1,798) and Medicare ($420.50) figures are correct, and it said Minnesota has no mandatory employee payroll tax. It never applied the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60, which PolicyEngine includes by assuming the employer withholds it." +us,scenario_032,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It stopped at employee FICA of $2,218.50 and said there was no mandatory Minnesota employee payroll tax. That left out the $127.60 Minnesota Paid Leave contribution (0.44% of wages), which PolicyEngine treats as withheld from the employee." +us,scenario_032,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It applied only the 7.65% federal FICA rate to $29,000. It never considered the 2026 Minnesota Paid Leave employee contribution of 0.44% ($127.60), which PolicyEngine counts as employee state payroll tax." +us,scenario_032,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no employee-side payroll tax beyond the federal ones. That missed the Minnesota Paid Leave premium starting in 2026, where PolicyEngine withholds the 0.44% employee share ($127.60) on the head's $29,000 in wages." +us,scenario_032,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It reported $2,218.50 and said no state disability or paid family leave employee contribution applied in this scenario. Minnesota Paid Leave premiums began January 1, 2026; the frozen reference assumes a $127.60 employee deduction, but the employer may pay that share itself and the prompt does not specify its choice." +us,scenario_032,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It assumed no mandatory Minnesota employee state payroll tax was modeled and answered $2,218.50. The frozen reference assumes a $127.60 Minnesota Paid Leave deduction (0.44% of wages); the employer may pay it instead, and the unspecified choice also permits federal tax alone, $2,218.50." +us,scenario_032,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said no employee state payroll tax applies and stopped at 7.65% FICA. That left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60, which PolicyEngine includes." +us,scenario_032,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It added a Minnesota Paid Leave share at 0.35% ($101.50), giving $2,320. The frozen reference assumes the maximum deduction, half of the 0.88% premium or 0.44% of wages ($127.60), so its answer is $26.10 below that reference; the employer may deduct less or pay the share itself, and the prompt does not specify its choice." +us,scenario_032,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It submitted $2,320, which matches a 0.35% Minnesota Paid Leave share ($101.50) on top of FICA, while its explanation says no state payroll tax applies and gives $2,218.50. The frozen reference adds the maximum deductible employee share, half of the 0.88% premium or 0.44% of wages ($127.60), for $2,346.10; the employer may pay that share itself, and the prompt does not specify its choice." +us,scenario_032,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It applied only 7.65% FICA to $29,000. It left out the Minnesota Paid Leave employee contribution (0.44% × $29,000 = $127.60), which PolicyEngine counts as employee state payroll tax." +us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It computed only federal FICA, rounded to $2,219. It never added the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60, which PolicyEngine includes." +us,scenario_032,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It computed 0.0765 × $29,000 = $2,218.50 and ignored state payroll taxes. That left out the Minnesota Paid Leave employee contribution of 0.44% ($127.60), which PolicyEngine adds by assuming the employer withholds it." +us,scenario_032,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted only the 7.65% FICA employee share. It left out the Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60, which PolicyEngine includes in payroll_tax." +us,scenario_032,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It reported only Social Security and Medicare tax ($2,218.50). It left out the 2026 Minnesota Paid Leave employee contribution of $127.60 (0.44% of $29,000), which PolicyEngine treats as withheld." +us,scenario_032,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA only. It left out the Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60, which PolicyEngine includes as mandatory employee state payroll tax." +us,scenario_032,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It stopped at 7.65% FICA ($2,218.50). It never applied the 0.44% Minnesota Paid Leave employee contribution ($127.60), which PolicyEngine adds for 2026." +us,scenario_032,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Its Social Security ($1,798) and Medicare ($420.50) figures are correct, but it left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60. PolicyEngine includes that contribution in payroll_tax." +us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax. The frozen reference is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave = $2,346.10, assuming the maximum employer deduction; the employer may pay that share itself, giving $2,218.50, and the prompt does not specify its choice." +us,scenario_032,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA and correctly ruled out Additional Medicare Tax, but it never considered state payroll taxes. That left out the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Its explanation lists only the 6.2% and 1.45% employee rates on $29,000 ($2,218.50), yet it submitted $4,428.60. That is roughly the combined employer-plus-employee 15.3% FICA rate and contradicts its own reasoning. It also left out the $127.60 Minnesota Paid Leave employee contribution." +us,scenario_032,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It correctly computed 7.65% × $29,000 = $2,218.50, then dropped it for a made-up 5.24% rate. It justified that with a nonexistent 'capped Social Security wage base interaction', even though $29,000 is far below the wage base. The frozen reference assumes a $127.60 Minnesota Paid Leave deduction (0.44% of wages); the employer may pay it instead, and the unspecified choice also permits federal tax alone, $2,218.50." +us,scenario_032,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It said no state payroll tax applies and stopped at $2,218.50 in FICA. That left out the Minnesota Paid Leave employee contribution (0.44% × $29,000 = $127.60), which PolicyEngine includes." +us,scenario_032,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA (rounded to $2,219) and ignored state payroll taxes. That left out the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It reported federal tax alone, $2,218.50, saying no other employee-side payroll tax was indicated. The frozen reference assumes a $127.60 Minnesota Paid Leave deduction (0.44% of wages); the employer may pay it instead, and the unspecified choice also permits federal tax alone, $2,218.50." +us,scenario_032,payroll_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It computed only 7.65% FICA on $29,000. It left out the Minnesota Paid Leave employee contribution of 0.44% ($127.60), which PolicyEngine counts as employee state payroll tax." +us,scenario_032,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied only the 6.2% Social Security and 1.45% Medicare rates. It left out the 2026 Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It said no mandatory Minnesota employee payroll tax applies. That missed the Minnesota Paid Leave premium, whose 0.44% employee share ($127.60) PolicyEngine treats as withheld from the head's wages." +us,scenario_032,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It said Minnesota has no employee-side payroll tax. Minnesota Paid Leave, effective 2026, has a 0.44% employee contribution ($127.60 on $29,000), which PolicyEngine adds to FICA." +us,scenario_032,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"It recognized Minnesota Paid Leave but excluded it because the employer can choose whether to withhold the employee share. PolicyEngine assumes the maximum 0.44% employee share is withheld ($127.60), and the prompt does not say which choice the employer makes." +us,scenario_032,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It computed FICA of $2,218.50 and never checked for state payroll taxes. That left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It said no state employee payroll taxes apply. That left out the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60, which PolicyEngine includes." +us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax. The frozen reference is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave = $2,346.10, assuming the maximum employer deduction; the employer may pay that share itself, giving $2,218.50, and the prompt does not specify its choice." +us,scenario_032,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It computed FICA of $2,218.50, rounded it to $2,219, and considered no state payroll tax. That left out the Minnesota Paid Leave employee contribution (0.44% × $29,000 = $127.60)." +us,scenario_032,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA only. It left out the 2026 Minnesota Paid Leave employee share of 0.44% × $29,000 = $127.60, which PolicyEngine counts in payroll_tax." +us,scenario_032,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It said Minnesota levies no mandatory employee payroll tax. That missed the Minnesota Paid Leave premium starting in 2026, whose 0.44% employee share ($127.60) PolicyEngine treats as withheld." +us,scenario_032,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It ruled out Minnesota payroll tax because Minnesota has no CA/NY-style disability insurance tax. That overlooked the Minnesota Paid Leave program, whose 0.44% employee share ($127.60) PolicyEngine includes." +us,scenario_032,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It submitted 7.65% FICA ($2,218.50); its text misstated Medicare as $419.50 instead of $420.50. It left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Placed income in the federal 130–185% FPL reduced-price band but ignored Minnesota's universal free school meals program. That program puts the household in the FREE tier, which supersedes reduced-price eligibility, so the correct value is 0." +us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied the federal 185% FPL reduced-price test and missed that Minnesota provides universal free meals, which puts the household in the FREE tier instead of reduced-price. It also miscounted the household as 4 people with 2 children when there are 3 people and 1 child." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Concluded reduced-price eligibility from income falling between 130% and 185% FPL, overlooking Minnesota's universal free school meals law. Under that law every student gets free meals, so the household's tier is FREE and reduced-price eligibility is 0." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"Used only the federal 130–185% income band and never applied Minnesota's universal free school meals program. That program classifies the household as FREE tier, so reduced-price support is zero." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Treated an income near 170% FPL as decisive for reduced-price meals and ignored that Minnesota gives all students free meals regardless of income. The household is in the FREE tier, not reduced-price." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,Applied the federal 130–185% reduced-price band to a 160% FPL income and missed Minnesota's universal free school meals. Those meals put the household in the FREE tier and make reduced-price eligibility 0. +us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,Repeatedly checked federal FPL thresholds for a 3-person household and never considered Minnesota's universal free school meals program. That program assigns the FREE tier to every student and supersedes the reduced-price classification. +us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Found income above 130% but below 185% FPL and called it reduced-price, missing that Minnesota's universal free meals put the household in the FREE tier. Reduced-price eligibility is therefore 0." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used the federal income bands, putting the household at about 157% FPL, without applying Minnesota's universal free school meals. Those meals give the child the FREE tier, which supersedes reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Tested only income against the 185% FPL reduced-price ceiling and ignored that Minnesota provides free meals to all students. That places the household in the FREE tier, so reduced-price eligibility is 0." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,Applied the federal 130–185% FPL band and missed Minnesota's universal free school meals program. Under that program the household is FREE tier rather than reduced-price. +us,scenario_032,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Classified the child as reduced-price from the federal 130–185% income band, overlooking Minnesota's universal free meals. Those meals assign the FREE tier and override reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Used the federal 130–185% FPL test and ignored Minnesota's universal free school meals program. That program makes the household's tier FREE, so reduced-price eligibility is 0." +us,scenario_032,reduced_price_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,Placed income at about 153% FPL in the reduced-price band but never applied Minnesota's universal free school meals. Those meals put the household in the FREE tier instead. +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,Relied on the federal free and reduced-price income limits and missed that Minnesota provides universal free meals regardless of income. The household is therefore FREE tier with no reduced-price support. +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"Applied the federal reduced-price income threshold without Minnesota's universal free school meals override. That override classifies the household as FREE tier, so reduced-price eligibility is 0." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,Used the federal reduced-price income range and ignored that Minnesota gives free meals to all students. That makes the household FREE tier rather than reduced-price. +us,scenario_032,reduced_price_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"Placed income in the federal reduced-price range and missed Minnesota's universal free school meals program. Under that program the household gets the FREE tier, which supersedes reduced-price." +us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,Compared income at about 156% FPL to the 185% reduced-price limit without applying Minnesota's universal free school meals. Those meals put the household in the FREE tier and make reduced-price eligibility 0. +us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,Used the federal 130–185% FPL band and never applied Minnesota's universal free meals program. That program classifies the household as FREE tier rather than reduced-price. +us,scenario_032,reduced_price_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,Applied the federal income band to a school-age child and missed that Minnesota provides free meals to all students. The FREE tier supersedes reduced-price eligibility. +us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Tested income at about 156.7% FPL against the 185% reduced-price limit only, ignoring Minnesota's universal free school meals. Those meals assign the FREE tier, so reduced-price eligibility is 0." +us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,state_local_rule,False,Used the federal 130–185% FPL dollar band and missed Minnesota's universal free school meals program. That program puts the household in the FREE tier instead of reduced-price. +us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,Returned no value or explanation for reduced_price_school_meals_eligible. The correct answer is 0 because Minnesota's universal free school meals put the household in the FREE tier. +us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"Placed income at about 156% of poverty in the federal reduced-price band but ignored Minnesota's universal free school meals. Those meals give the household the FREE tier, which supersedes reduced-price." +us,scenario_032,reduced_price_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"Applied the federal free and reduced-price income limits without Minnesota's universal free meals override. That override classifies the household as FREE tier, so reduced-price eligibility is 0." +us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Compared income at about 160% FPL to the 185% reduced-price cutoff and missed that Minnesota provides universal free school meals. The household is FREE tier, not reduced-price." us,scenario_032,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_032,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly used Minnesota broad-based categorical eligibility to bypass the failed SNAP gross-income screen, then contradicted its own calculation showing that net income produced a negative allotment. It replaced that result with an unsupported $493 monthly benefit despite no listed shelter expense or other deduction capable of generating it." us,scenario_032,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota broad-based categorical eligibility as sufficient to pass the household through the gross-income screen. Its own max-allotment-minus-30%-of-net-income calculation was negative, but it then invented a $476 monthly benefit; SNAP's minimum benefit does not convert a negative allotment for this ineligible three-person household into that amount." @@ -3109,53 +3157,54 @@ us,scenario_032,state_income_tax_before_refundable_credits,glm-5.3,llm_error,sta us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model asserted that Minnesota AGI equals federal AGI ""with no modifications,"" discarding the Minnesota Social Security subtraction that removes the entire $1,916 of federally taxable benefits at this income, and it also omitted the Minnesota dependent exemption of roughly $5,300 for Child 1. Applying that subtraction against its own $30,699 standard deduction gives $29,000 − $30,699 < 0, so taxable income is zero and the tax is $0, not the $217 base it taxed at 5.35%." us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs object. This is a failure to deliver the answer under the output contract, not a substantive error in the Minnesota computation." us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model declared the full $13,664 of Social Security ""fully taxable"" for Minnesota purposes, contradicting the §86 provisional-income formula (provisional income $35,832 against the $32,000 joint base makes only $1,916 taxable) and the Minnesota Social Security subtraction that removes even that $1,916, and it used a $28,500 joint standard deduction below Minnesota's 2026 amount with no dependent exemption. Its own stated chain of $42,664 − $28,500 at 5.35% yields $758, not the $128 it submitted, so the figure is an unsupported approximation layered on an inflated income base; the correct $29,000 base falls below the deduction and yields $0." -us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Added a separate ~$925 one-child 'working family credit component' to the $1,750 child amount. Under the CWFC, a child under 18 generates only the child amount, and the working-family part is just 4% of earnings up to the base cap. That overstates the total compared with the $2,187.60 combined credit." -us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Added a ~$950 'one-child add-on' to the working-family component. The CWFC gives extra working-family amounts only for qualifying children aged 18 to 24, not for a 6-year-old. The model's ~$3,100 therefore far exceeds the $2,187.60 credit, which is the child amount plus the capped 4%-of-earnings base." -us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Stated that Minnesota has no refundable individual income tax credits. This misses the fully refundable Child and Working Families Credit, which pays $2,187.60 for this family with one child aged 6, $29,000 of earnings, and AGI below the phase-out threshold." -us,scenario_032,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Found no Minnesota refundable credit and missed the Child and Working Families Credit. That credit pays the indexed per-child amount for the 6-year-old plus the capped working-family component, for $2,187.60 in total." -us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Kept the child amount at the unindexed 2023 figure of $1,750 and guessed $410 for the working-family part. It should have used the 2026 inflation-adjusted CWFC amounts, which combine to $2,187.60 with no phase-out, so its $2,160 is too low." -us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Reported only the unindexed $1,750 child amount and dropped the working-family part of the combined CWFC, even though $29,000 of earnings maxes out that part. It also missed the 2026 inflation adjustment, which brings the combined credit to $2,187.60." -us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Reduced the working-family part by a 'shared phaseout' that does not apply, because AGI of $30,916 is below the 2026 married-filing-jointly CWFC threshold. It also estimated rather than computing the indexed child amount, so it landed at $2,185 instead of the unreduced $2,187.60." -us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Used the 2023 statutory amounts ($1,750 child + $350 working family = $2,100) without the inflation indexing Minnesota applies to the CWFC from 2024 onward, which raises the 2026 credit to $2,187.60." -us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Calculated the pre-2023 Working Family Credit as about 35% of the federal EITC and left out the child credit. In 2023, Minnesota replaced that structure with the CWFC: an indexed per-child amount for the 6-year-old plus a working-family part of 4% of earnings, capped, for $2,187.60 in total." -us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Assumed Minnesota's refundable credits require rent, property tax or education expenses. It missed that the CWFC is based on earnings and a qualifying child alone, giving this family with a child under 18 and $29,000 of wages $2,187.60." -us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Stacked a ~$1,900 old-style, EITC-based Working Family Credit on top of the child credit. Under the CWFC, the working-family part for a family with no qualifying children aged 18 to 24 is only the capped 4%-of-earnings base, so the correct total is $2,187.60, not $3,700." -us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Calculated the working-family part as 45% of an overstated $4,625 federal EITC using the repealed EITC-linked formula, then added $1,750. Under the CWFC, that part is only 4% of earnings, capped at the base, plus the indexed child amount, for $2,187.60." -us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Added a working-family amount equal to 33% of the federal EITC ($1,481) to the child credit. That pre-2023 formula no longer exists, because the CWFC's working-family part is the capped 4%-of-earnings base. This overstated the credit at $3,393 instead of $2,187.60." -us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Assigned a $1,633 Working Family Credit using the old child-based WFC schedule on top of a $1,950 child credit. Under the CWFC, the working-family part for a child under 18 is only the capped 4%-of-earnings base, so the combined credit is $2,187.60." -us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Computed a $1,368 working-family amount as 45% of the federal EITC, which is the repealed pre-2023 formula, and added the unindexed $1,750. The CWFC is the indexed child amount plus a working-family part of 4% of earnings, capped, for $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Cut the child credit to $983.60 with a phase-out even though AGI of $30,916 is below the married-filing-jointly threshold, so no reduction applies. It also set the working-family part at $550, above its cap. Its $1,533.60 understates the full $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that no Minnesota refundable credit applies. This misses the CWFC, which pays $2,187.60 to a married-filing-jointly household with a child aged 6, $29,000 of earnings, and AGI below the phase-out threshold." -us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Doubled the working-family base to $700 'for the taxpaying couple'. The CWFC's working-family part is one capped 4%-of-earnings amount per return, not per spouse. The model also left the child amount unindexed at $1,750, giving $2,450 instead of $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Added a ~$1,200 Working Family Credit sized like the old EITC-linked one-child credit. Under the CWFC, the working-family part for a family whose only child is 6 is just the capped 4%-of-earnings base, so the total is $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $600 is consistent with counting only a small working-family-type amount and leaving out the CWFC child amount for the 6-year-old. That child amount alone is about $1,750 (2023 base, indexed for 2026), and the full combined credit is $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Counted only the unindexed $1,750 child amount. It left out the CWFC's working-family part, which is maxed out at $29,000 of earnings, and the 2026 inflation adjustment that together bring the credit to $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Added an implied $1,085 Working Family Credit to the $1,750 child credit. Under the CWFC, a family with no children aged 18 to 24 gets only the capped 4%-of-earnings working-family base. That overstated the total at $2,835 instead of $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $2,450 matches $1,750 plus a doubled $700 working-family base, which means it counted the base once per spouse. The CWFC allows one capped 4%-of-earnings amount per return, plus the 2026 indexed child amount, for $2,187.60." -us,scenario_032,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Inflated the child amount at an assumed 3% a year to $1,912 but stated that no other refundable credit applies. That leaves out the working-family part of the same CWFC (4% of $29,000 of earnings, capped), which brings the combined credit to $2,187.60." -us,scenario_032,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Applied the $29,500 phase-out threshold for non-joint filers to this married-filing-jointly household. The joint threshold is higher than $30,916 of AGI, so no phase-out applies. The model also treated the working-family part as near its phase-out end instead of at its capped base." -us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no Minnesota refundable credit and overlooked the CWFC. That credit is the indexed child amount for the 6-year-old plus the capped working-family part on $29,000 of earnings, for $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Treated refundable credit eligibility as unspecified and returned $0, even though the household facts (a qualifying child aged 6, $29,000 of wages and AGI below the threshold) are enough to compute the $2,187.60 CWFC." -us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used the 2023 base amounts ($1,750 child + $350 working family = $2,100) and missed the inflation indexing Minnesota applies to the CWFC after 2023, which raises the 2026 credit to $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Calculated the working-family amount as 25% of the federal EITC ($1,106.75), which is a repealed EITC-linked formula. The CWFC's working-family part is the capped 4%-of-earnings base, so the combined credit is $2,187.60, not $2,856.75." -us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Assigned a $1,675 Working Family Credit using the old child-based WFC schedule. Under the CWFC, a child under 18 generates only the child amount, and the working-family part is the capped 4%-of-earnings base, so the total is $2,187.60." -us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $3,572 is consistent with the child credit plus an old-style, EITC-linked one-child WFC of about $1,800. Under the CWFC, the working-family part is limited to the capped 4%-of-earnings base, so the combined credit is $2,187.60." -us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Left the child amount at the unindexed 2023 figure of $1,750 and used $388 for the working-family part. The 2026 inflation-adjusted CWFC amounts combine to $2,187.60, so its $2,138 is too low." -us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Its $4,500 is consistent with a large EITC-style Working Family Credit stacked on the child credit. Under the CWFC, the working-family part for a family whose only child is 6 is just the capped 4%-of-earnings base, giving $2,187.60." -us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Added a $1,275 Working Family Credit based on the repealed child-based WFC schedule. Under the CWFC, the working-family part is the capped 4%-of-earnings base and the child amount is indexed, for $2,187.60." -us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Left the child amount at the unindexed 2023 figure of $1,750 and used $387 for the working-family part. The 2026 inflation-adjusted CWFC amounts combine to $2,187.60." -us,scenario_032,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Counted only a ~$500 WFC-style amount and left out the CWFC child amount for the 6-year-old. That child amount is the largest part of the $2,187.60 combined credit." -us,scenario_032,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Calculated the Working Family Credit as 40% of the federal EITC ($1,774), which is the repealed pre-2023 formula. The CWFC's working-family part is the capped 4%-of-earnings base plus the indexed child amount, for $2,187.60." -us,scenario_032,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Estimated a $1,560 Working Family Credit using the old one-child WFC schedule on top of the $1,750 child credit. The CWFC limits the working-family part to the capped 4%-of-earnings base when the only child is under 18, so the total is $2,187.60." -us,scenario_032,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Used a $1,577 'one-child maximum' Working Family Credit, which is the old WFC schedule. Under the CWFC, children under 18 generate only the child amount and the working-family part is the capped 4%-of-earnings base, giving $2,187.60." -us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Computed the Working Family Credit as 33% of the federal EITC ($1,465), which is a formula Minnesota repealed when it created the CWFC. The working-family part is now the capped 4%-of-earnings base plus the indexed child amount, for $2,187.60." -us,scenario_032,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Added an estimated $1,300 one-child Working Family Credit to the $1,750 child credit. The CWFC gives no child-based working-family increase for a child under 18, so the credit is the indexed child amount plus the capped 4%-of-earnings base, $2,187.60." -us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not return a usable value or explanation for state_refundable_credits, so no answer could be scored against the $2,187.60 CWFC." -us,scenario_032,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Used a $1,250 'full one-child' Working Family Credit from the old WFC schedule. The CWFC's working-family part is the capped 4%-of-earnings base, and combined with the indexed child amount the credit is $2,187.60." -us,scenario_032,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Denied the credit because the spouse had no earnings and income was supposedly above the thresholds. The CWFC requires earnings from only one spouse, and AGI of $30,916 is below the married-filing-jointly phase-out threshold. The model also never considered the child amount for the 6-year-old, so it missed the full $2,187.60." -us,scenario_032,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Modeled the Working Family Credit with a ~$2,900 one-child maximum and a 12% phase-out, which mixes the old WFC schedule with CWFC rules. The working-family part for this family is only the capped 4%-of-earnings base, and AGI is below the phase-out threshold, so the credit is $2,187.60." -us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Calculated only an EITC-percentage Working Family Credit (~$1,895) and left out the child amount, which is the main part of the CWFC. The correct credit is the indexed child amount plus the capped 4%-of-earnings base, $2,187.60." -us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Used a $1,648 old-schedule Working Family Credit and added a $241 dependent care credit even though the household has no childcare expenses. It also left out the CWFC child amount, which together with the capped 4%-of-earnings base makes up the $2,187.60." +us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used a working-family component of about $925, which comes from the repealed EITC-style WFC schedule. The CWFC working-family piece is 4% of earnings capped at about $365 (indexed). It also applied a vague reduction even though AGI of $30,916 is below the joint phaseout threshold. The correct total is $2,187.60, not $2,337." +us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It added a one-child working-family add-on of about $950 from the pre-2023 WFC structure. Under the CWFC, the working-family amount is only 4% of earnings capped at about $365, with no per-child add-on for a child under 19. The correct total is about $1,823 + $365 = $2,187.60, not $3,100." +us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It stated that Minnesota has no refundable individual income tax credits. In fact, Minnesota's Child and Working Families Credit is fully refundable. This household has one qualifying 6-year-old and AGI below the phaseout threshold, so it receives $2,187.60." +us,scenario_032,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It found no Minnesota refundable credit and missed the fully refundable Child and Working Families Credit. With one qualifying child and $29,000 of earnings, the household receives the full indexed child amount plus the capped 4% working-family amount: $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the correct CWFC structure but did not index the $1,750 child amount to 2026 (about $1,823). It also set the working-family cap at $410 instead of the indexed cap of about $365 (base $350). That gave $2,160 instead of $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It counted only the unindexed $1,750 child amount. It dropped the working-family component (4% of $29,000, capped at about $365) that is part of the same refundable CWFC, and it did not index the child amount to 2026. The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It took the unindexed $1,750 child amount and then reduced the working-family portion through a shared phaseout it assumed began around $31,000. The joint-filer CWFC threshold is well above the $30,916 AGI, so no phaseout applies. The full indexed amounts give $2,187.60, not $2,185." +us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It used the right components (child amount plus 4% of earnings, capped) but at their unindexed base values of $1,750 and $350. It did not apply the inflation indexing to 2026, which raises the total to $2,187.60." +us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It modeled the MN credit as the repealed WFC, a percentage of the federal EITC, and left out the child amount entirely. The 2023 CWFC replaced that structure with a per-child amount plus 4% of earnings capped at about $365, which totals $2,187.60. It also overstated AGI by treating 85% of the Social Security benefits as taxable, when only $1,916 is taxable." +us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It concluded that refundable MN credits require rent, property tax, or education expenses, and so missed the Child and Working Families Credit. That credit requires only a qualifying child and earned income. This household qualifies for the full $2,187.60." +us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It added a working-family credit of about $1,900 from the old EITC-style WFC schedule. The CWFC working-family amount is 4% of earnings capped at about $365. With the indexed child amount of about $1,823, the correct total is $2,187.60, not $3,700." +us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It computed the WFC as 45% of the federal EITC, which is the repealed pre-2023 formula, and stacked it on a $1,750 child credit. The CWFC working-family piece is 4% of earnings capped at about $365. The correct total is $2,187.60, not $3,831." +us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It over-indexed the child amount to $1,912 and added a WFC equal to 33% of the federal EITC, a formula Minnesota no longer uses. The CWFC is about $1,823 for the child plus 4% of earnings capped at about $365, for $2,187.60." +us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $1,633 WFC from the old EITC-linked schedule and a $1,950 child credit, overstating both pieces. Under the CWFC, the indexed child amount is about $1,823 and the working-family cap is about $365, for $2,187.60." +us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It computed the WFC as 45% of the federal EITC ($1,368), which is the repealed formula, and added it to $1,750. The CWFC working-family amount is 4% of earnings capped at about $365. The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It phased the child amount down to $983.60 even though the household's $30,916 AGI is below the joint-filer CWFC phaseout threshold. It also used a $550 working-family amount instead of the cap of about $365. With no phaseout, the full credit is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It found no refundable credits and missed Minnesota's Child and Working Families Credit. The household has a qualifying 6-year-old and earnings under the phaseout threshold, so it receives $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It doubled the working-family amount to $700 by treating it as a per-taxpayer credit for the couple. The CWFC working-family amount is a single per-return cap (about $365 indexed), and the child amount is also indexed (about $1,823). The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It estimated a working-family credit of about $1,200 using the old EITC-style WFC magnitudes. The CWFC working-family amount is 4% of earnings capped at about $365. With the indexed child amount, the correct total is $2,187.60, not $2,950." +us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Its $600 answer is consistent with counting only a small working-family credit and leaving out the child amount (about $1,823). The correct total, with the child amount plus the working-family cap of about $365, is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It counted only the unindexed $1,750 child credit. It omitted the working-family component (4% of earnings, capped at about $365) and the 2026 inflation indexing, which together bring the CWFC to $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It added a working-family credit of $1,085 to the $1,750 child credit. The CWFC working-family amount is capped at about $365 (4% of earnings, base $350). The correct total is $2,187.60, not $2,835." +us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"Its $2,450 is consistent with $1,750 plus a doubled $700 working-family amount, applied per spouse instead of once per return. The indexed CWFC is about $1,823 plus about $365, for $2,187.60." +us,scenario_032,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It over-indexed the child amount to $1,912 and said no other refundable credit applies. That missed the CWFC working-family component of 4% of earnings, capped at about $365. The correct total is about $1,823 + $365 = $2,187.60." +us,scenario_032,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It applied the $29,500 phaseout threshold for non-joint filers and reduced the child amount by about $170. This household files jointly, and its $30,916 AGI is below the joint threshold, so no phaseout applies. It also understated the working-family amount at $180 instead of the full cap of about $365. The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It identified no refundable MN credit and missed the Child and Working Families Credit. The household qualifies through its 6-year-old dependent and $29,000 of earnings and receives $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It treated refundable credits as absent because no explicit credit amounts were listed. Minnesota's CWFC is computed from the listed facts (one qualifying child, $29,000 of earnings, AGI below the phaseout threshold) and equals $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used the correct CWFC components at their unindexed base values, $1,750 child amount and $350 working-family cap. It did not apply the 2026 inflation indexing, which raises the total to $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It computed the working-family credit as 25% of a federal EITC, a formula Minnesota abandoned in 2023. The CWFC working-family piece is 4% of earnings capped at about $365. The correct total is $2,187.60, not $2,856.75." +us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used a $1,675 Working Family Credit from the old EITC-style schedule. The CWFC caps the working-family amount at about $365. With the indexed child amount, the correct total is $2,187.60, not $3,425." +us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $3,572 is consistent with a $1,750 child credit plus an old-style EITC-linked WFC of about $1,800. The CWFC working-family component is 4% of earnings capped at about $365, for a total of $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used the unindexed $1,750 child amount instead of the 2026 indexed amount of about $1,823, and set the working-family cap at $388 instead of about $365. That gave $2,138 instead of $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Its $4,500 overstates the CWFC by more than double, consistent with stacking an old EITC-sized WFC on the child credit. The working-family piece is capped at about $365, for a total of $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It added a $1,275 working-family credit from the repealed EITC-linked structure to $1,750. The CWFC working-family cap is about $365 and the indexed child amount is about $1,823, for $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It used the unindexed $1,750 child amount instead of about $1,823 for 2026, and a $387 working-family amount instead of the indexed cap of about $365. That gave $2,137 instead of $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It counted only a working-family credit of about $500 and left out the child amount of the CWFC (about $1,823 indexed) for the qualifying 6-year-old. The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It computed the WFC as 40% of the federal EITC ($1,774), which is the repealed pre-2023 formula. The CWFC working-family piece is 4% of earnings capped at about $365. The correct total is $2,187.60, not $3,524." +us,scenario_032,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It added a $1,560 working-family credit from the old EITC-style schedule to $1,750. The CWFC caps the working-family amount at about $365, so the correct total is $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a one-child WFC maximum of $1,577 from the pre-2023 schedule. The CWFC has no child-based WFC tier for a 6-year-old; its working-family amount is 4% of earnings capped at about $365. The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It computed the WFC as 33% of the federal EITC ($1,465), which is the repealed formula, and added $1,750. The CWFC working-family cap is about $365, giving $2,187.60." +us,scenario_032,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"It estimated a one-child WFC of about $1,300 from the old EITC-linked schedule. The CWFC working-family amount is 4% of earnings capped at about $365. With the indexed child amount of about $1,823, the correct total is $2,187.60." +us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no parseable value or explanation for state_refundable_credits, so no answer was scored against the $2,187.60 CWFC." +us,scenario_032,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used a full one-child WFC of $1,250 from the old schedule. Under the CWFC, the working-family amount is 4% of earnings capped at about $365. With the indexed child amount, the correct total is $2,187.60." +us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It wrongly concluded that household income exceeds the eligibility thresholds and that the spouse's lack of earnings blocks the credit. The CWFC uses the head's $29,000 of earnings, and the $30,916 AGI is below the joint phaseout threshold. The full $2,187.60 applies." +us,scenario_032,state_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It estimated an $870 working-family credit. The CWFC working-family amount is 4% of earnings capped at about $365, and the child amount is indexed to about $1,823. The correct total is $2,187.60, not $2,620." +us,scenario_032,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It assumed a WFC maximum of about $2,900 from the old schedule and reduced it by a phaseout to $1,650. The CWFC working-family cap is about $365, and no phaseout applies below the joint threshold. The correct total is $2,187.60." +us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It computed only an EITC-percentage WFC of $1,895 and never included the CWFC child amount for the 6-year-old. The current credit is about $1,823 for the child plus 4% of earnings capped at about $365, for $2,187.60." +us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $1,648 WFC from the old EITC-linked schedule and added a $241 dependent care credit even though the household has no care expenses. It also omitted the CWFC child amount. The correct total is about $1,823 + $365 = $2,187.60." us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $243 state tax refund as non-taxable, which put AGI at $83,208 instead of $83,451.23 and taxable income at $35,708. It then threw out its own $3,789 result and reported an unexplained $3,699. Including the refund gives $35,951.23 of taxable income and $3,818.15 of tax." us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI and the $47,500 standard-plus-senior deduction match, but it also took the $1,225 auto loan interest as a deduction. That deduction doesn't apply to this household. The extra $1,225 × 12% = $147 explains its $3,671.12 versus $3,818.15." us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly treated the $11,400 self-employment loss as a capital loss capped at $3,000, when the full amount offsets ordinary income. It also used only a $30,400 basic standard deduction, leaving out the $3,300 aged additions and the $12,000 senior deduction. It then reported $5,661, which doesn't follow from its own calculation." @@ -3207,72 +3256,74 @@ us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"It explicitly deri us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed $1,989 + $4,054.50 = $6,043.50, then 'rounded' to $6,042. That moves the value in the wrong direction and away from the exact amount it had already found." us,scenario_033,payroll_tax,qwen3.8-max,llm_error,other,False,"It named the correct rates (6.2% + 1.45%) and the correct $79,000 base, but its $6,123 equals 7.75% × $79,000. It multiplied with a mistaken combined rate instead of 7.65%, which gives $6,043.50." us,scenario_033,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly recognized that a self-employment loss produces no self-employment tax, then contradicted that rule by reporting an invented negative liability. A negative SSTB amount cannot be multiplied by the 15.3% rate to create a self-employment tax credit; the correct liability is floored at zero." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the right result ($1,240 + $3,686 = $4,926 on $43,120 of taxable income) but then submitted $4,849, citing unspecified 'projected bracket indexing'. That number has no basis in its own correct 2026 parameters." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the old $15,000 standard deduction instead of the 2026 OBBBA amount of $16,100. Its arithmetic also doesn't hold together: it cites a $3,809 figure and 12% over $11,600, then reports $8,074, which is far above any 10%/12% bracket calculation on about $44,220 of taxable income. The correct tax on $43,120 is $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model used the right $16,100 deduction but never applied the brackets explicitly. It guessed '$4,950' instead of computing 10% of $12,400 plus 12% of ($43,120 − $12,400), which is $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model overstated the 2026 single standard deduction as $16,600 instead of $16,100, which gave $4,866. It then cut that by an arbitrary 'bracket inflation' adjustment to $4,297, which its own calculation doesn't support." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly computed $1,240 + $3,686 = $4,926 on $43,120 of taxable income, then 'rounded to bracket estimate' and submitted $4,969. That adjustment departs from its own correct result." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly derived $4,926 from the $16,100 deduction and the $12,400/$50,400 brackets, then submitted $5,194 as an unexplained 'rounded estimate'. Nothing in its own calculation supports that number." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $15,750 (the 2025 OBBBA amount) as the 2026 standard deduction and $11,950 as the top of the 10% bracket. The correct 2026 figures are $16,100 and $12,400. That overstated taxable income at $43,470 and gave $4,977 instead of $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model computed exactly $4,926.40 ($1,240 + $3,686.40 on $43,120) and then submitted $6,499. That value contradicts its own correct derivation." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model's taxable income of $43,820 implies a standard deduction of about $15,400 instead of the 2026 OBBBA amount of $16,100. Applying the 10%/12% brackets to that inflated base gave $5,020 instead of $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used 2025 pre-OBBBA parameters: a $15,000 standard deduction and a 10% bracket ending at $11,925. The correct 2026 figures are $16,100 and $12,400, which give $43,120 of taxable income and $4,926.40 of tax." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026 and used an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no exemption and 10%/12% brackets." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $15,600 standard deduction and a 10% bracket ending at $12,200. The correct 2026 figures are $16,100 and $12,400. Its errors raised taxable income to $43,620 and gave $4,990.40 instead of $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026 and used an $8,300 standard deduction, a $5,300 exemption and a 15% second bracket. OBBBA made the TCJA brackets and the higher standard deduction permanent, so the correct calculation is ($59,220 − $16,100) taxed at 10%/12%, which gives $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The $5,122 answer matches 2024 parameters exactly: a $14,600 standard deduction and a 10% bracket ending at $11,600, which give $1,160 + 12% × $33,020. The model needed the 2026 figures of $16,100 and $12,400, which give $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires and brought back an $8,350 standard deduction, a $5,050 personal exemption and the pre-TCJA 15% bracket. Under OBBBA, 2026 keeps a $16,100 single standard deduction with no exemption and 10%/12% brackets." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a TCJA expiry: an $8,300 standard deduction, a $5,050 exemption and a 15% rate above $11,600. OBBBA kept the TCJA structure permanently, so taxable income is $43,120 and tax is $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation and mentioned 'exemptions', which no longer exist after OBBBA. Its $3,858 understates the tax: $43,120 of taxable income ($59,220 − $16,100) taxed at 10% up to $12,400 and 12% above gives $4,926.40, and $3,858 would require a deduction of roughly $25,000." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used TCJA-expiry rules (an $8,300 standard deduction, a $5,300 exemption and a 15% bracket above $11,925), but OBBBA made the TCJA rates and the $16,100 standard deduction permanent for 2026." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed post-TCJA-expiry law, with an $8,300 standard deduction, a $5,300 personal exemption and a 15% rate. OBBBA extended the TCJA permanently, so the correct calculation uses the $16,100 deduction and 10%/12% brackets, which give $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction and a 10% bracket ending at $11,925. The correct 2026 figures are $16,100 and $12,400, which give $43,120 of taxable income and $4,926.40 of tax." -us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model explicitly used 2024 figures (a $14,600 standard deduction and a 10% bracket ending at $11,600) as stand-ins for 2026. The 2026 figures of $16,100 and $12,400 give $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model mixed a 2023 standard deduction ($13,850) with an invented medical-expense adjustment and 15%/22% brackets that don't exist under current law. Its taxable income of $50,630 is inconsistent with its own numbers. The correct calculation is $59,220 − $16,100 = $43,120, taxed at 10%/12%, which gives $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no bracket calculation. Its $3,344 understates the tax on $43,120 of taxable income, which under the 2026 brackets is $1,240 + 12% × $30,720 = $4,926.40. The figure is consistent with too large a deduction or a lower rate applied to part of the income." -us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model gave no bracket calculation and asserted $3,720. Applying the 2026 $16,100 standard deduction and the 10%/12% brackets (the 10% bracket ends at $12,400) to $59,220 gives $4,926.40, so the model understated the liability." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed 'deductions including mortgage interest', but the facts list only a mortgage balance. Mortgage interest is unlisted and therefore $0, so the $16,100 standard deduction applies, giving $43,120 of taxable income and $4,926.40 of tax, not $4,966." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a TCJA expiry (an $8,300 standard deduction, a $5,300 exemption and a 15% rate above $12,400). OBBBA kept the $16,100 standard deduction and the 12% bracket up to $50,400, which gives $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used 'current-law TCJA sunset' parameters: an $8,532 standard deduction, a $5,442 exemption and a 15% rate. Under OBBBA the TCJA structure is permanent, so 2026 uses a $16,100 deduction and 10%/12% brackets." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used the correct $16,100 deduction but ended the 10% bracket at $12,200 instead of the 2026 threshold of $12,400. That taxed an extra $200 at 12% instead of 10% and gave $4,930.40 instead of $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied 'pre-TCJA' 2026 rules: an $8,315 standard deduction, a $5,303 exemption and a 15% bracket. OBBBA made the TCJA brackets and the $16,100 single standard deduction permanent, which gives $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction and a 10% bracket ending at $11,925 'as a practical estimate'. The 2026 figures of $16,100 and $12,400 give $43,120 of taxable income and $4,926.40 of tax." -us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used $15,750 (the 2025 OBBBA amount) instead of the 2026 standard deduction of $16,100, and got $4,973. It then submitted an unexplained $6,437 that contradicts its own bracket calculation." -us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed income above $43,100 at 22%, although 12% runs to $50,400 in 2026. It claimed the elderly/disabled credit, which needs the filer to be 65 or older or permanently disabled, and the head is 56. It also added a 'medical expense deduction' to the tax, although medical costs here are below the 7.5% AGI floor and a deduction can't increase tax. That produced $8,440.50 instead of $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It computed the correct figure: $16,100 deduction, $43,120 taxable income and $1,240 + $3,686 = $4,926. It then dropped that result for an unsupported 'projected bracket indexing' figure of $4,849. The correct parameters were already in its own calculation." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a stale $15,000 standard deduction instead of the 2026 OBBBA amount of $16,100. Its bracket arithmetic is also incoherent: it applied 2024 thresholds and still got $8,074, which is far more than 10%/12% rates can produce on about $44,000 of taxable income. The correct result is $1,240 + 12% × $30,720 = $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It used the correct $16,100 deduction but rounded taxable income to about $43,100 and gave a loose $4,950. It never actually applied the brackets. Doing so gives $1,240 (10% up to $12,400) + 12% × $30,720 = $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It overstated the 2026 single standard deduction as $16,600 instead of $16,100 and got about $4,866. It then cut that by an unexplained 'bracket inflation' amount to $4,297. The correct result is $4,926.40 on $43,120 of taxable income." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived the correct $4,926 from the $16,100 deduction and the $12,400/$50,400 brackets. It then 'rounded' to $4,969 for no reason. The right answer was its own calculation, $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly computed $43,120 of taxable income and $4,926 of tax. It then replaced that with an unexplained $5,194. No other parameter applies, and the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $15,750 (the 2025 OBBBA amount) as the 2026 standard deduction instead of $16,100, and a 2025-level 10% bracket of about $11,950 instead of $12,400. That produced $43,470 of taxable income and $4,977. With 2026 parameters the tax is $1,240 + 12% × $30,720 = $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its reasoning computed exactly $4,926.40 using the right $16,100 deduction and $12,400/$50,400 brackets. It then submitted $6,499, which has no basis in its own work." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its $43,820 of taxable income implies a $15,400 standard deduction instead of the 2026 single amount of $16,100. It also used a lower 10% bracket threshold, giving $5,020 instead of $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 parameters: a $15,000 standard deduction and a 10% bracket ending at $11,925. The 2026 values are a $16,100 deduction and a 10% bracket ending at $12,400. That overstated taxable income at $44,220 and tax at $5,067.90." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA would sunset and applied an $8,300 standard deduction, a $5,300 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 deduction, no personal exemption and a 12% bracket up to $50,400. The tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It understated both 2026 parameters: a $15,600 deduction instead of $16,100, and a 10% bracket ending at $12,200 instead of $12,400. That produced $43,620 of taxable income and $4,990.40 instead of $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated 2026 as post-TCJA-sunset, with an $8,300 standard deduction, a $5,300 personal exemption and a 15% second bracket. OBBBA kept the $16,100 deduction, no exemption and the 12% bracket, so the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $5,122 matches 2024 parameters exactly: a $14,600 deduction and a 10% bracket ending at $11,600, giving $1,160 + 12% × $33,020. It did not use the 2026 values of a $16,100 deduction and a $12,400 10% bracket, which yield $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA would expire in 2026 and reinstated an $8,350 standard deduction, a $5,050 personal exemption and a 15% bracket. OBBBA made the TCJA rates and deduction permanent: a $16,100 deduction, no exemption and 12% on taxable income up to $50,400, giving $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the TCJA-sunset law: an $8,300 deduction, a $5,050 exemption and a 15% bracket. OBBBA kept the 2026 single standard deduction at $16,100 with no personal exemption and a 12% bracket, so the tax is $4,926.40, not $6,300.50." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,858 implies about $34,200 of taxable income, which means roughly $25,000 of deductions plus an exemption it said it applied. The only deduction available is the $16,100 standard deduction. There is no personal exemption and no itemizable mortgage interest, so taxable income is $43,120 and tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used post-TCJA-sunset parameters: an $8,300 deduction, a $5,300 exemption and a 15% bracket above $11,925. Under OBBBA, 2026 has a $16,100 deduction, no exemption and 12% up to $50,400, so the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used post-TCJA-sunset parameters: an $8,300 deduction, a $5,300 exemption and a 15% bracket above $11,925. Under OBBBA, 2026 has a $16,100 deduction, no exemption and 12% up to $50,400, so the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 parameters: a $15,000 deduction and a 10% bracket ending at $11,925. The 2026 values are a $16,100 deduction and a 10% bracket ending at $12,400. That overstated taxable income at $44,220 and tax at $5,067.90." +us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It explicitly used the 2024 $14,600 deduction and 2024 brackets ($11,600/$47,150) as stand-ins for 2026. With the 2026 values of a $16,100 deduction and a $12,400 10% bracket, taxable income is $43,120 and tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It mixed the 2023 $13,850 deduction with an invented medical adjustment and applied 15%/22% rates under 2025 rules. Medical costs here fall below the 7.5%-of-AGI floor, and the 2026 single brackets are 10% to $12,400 and 12% to $50,400 after a $16,100 deduction. That gives $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $3,344 implies only about $29,900 of taxable income, meaning roughly $29,000 of deductions. That is far above the $16,100 standard deduction, which is the only deduction available because no mortgage interest is listed. The correct taxable income is $43,120 and the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $3,720 implies about $33,000 of taxable income, meaning roughly $26,000 of deductions. Only the $16,100 standard deduction applies here, so taxable income is $43,120 and tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed a mortgage-interest deduction, but only a mortgage balance is listed and no interest paid. Itemizing therefore does not beat the $16,100 standard deduction. Its $4,966 does not match the correct $1,240 + 12% × $30,720 = $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA would sunset, applying an $8,300 deduction, a $5,300 exemption and a 15% rate above $12,400. OBBBA kept a $16,100 deduction, no exemption and a 12% bracket to $50,400, so the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied 'current-law TCJA sunset' parameters: an $8,532 deduction, a $5,442 exemption and a 15% bracket. OBBBA made the TCJA structure permanent, with a $16,100 deduction, no exemption and 12% to $50,400, giving $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the right $16,100 deduction and $43,120 taxable income. However, it set the 10% bracket end at $12,200 instead of the 2026 single threshold of $12,400. That gave $4,930.40 instead of $1,240 + 12% × $30,720 = $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA rules: an $8,315 deduction, a $5,303 exemption and a 15% bracket. OBBBA keeps a $16,100 standard deduction with no exemption and a 12% bracket in 2026, so the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It explicitly used the 2025 $15,000 deduction and the 2025 $11,925 10% bracket as proxies for 2026. The 2026 values are a $16,100 deduction and a $12,400 10% bracket, giving $43,120 of taxable income and $4,926.40 of tax." +us,scenario_036,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It identified the correct $16,100 deduction and $43,120 taxable income but never finished the bracket arithmetic, and submitted $4,936. The exact calculation is $1,240 + 12% × $30,720 = $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used $15,750 (the 2025 amount) instead of the 2026 standard deduction of $16,100, and a $12,150 10% bracket instead of $12,400, and got about $4,973. It then arbitrarily raised that to $6,437. The correct tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It invented an elderly credit, which requires age 65 or older or a disability, for a 56-year-old. It also added a medical-expense 'item' on top of the tax, which is not how tax is computed. The tax on $43,120 of taxable income is just $1,240 + 12% × $30,720 = $4,926.40, with no credits." us,scenario_036,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly declared the $59,220 pension income below a New Jersey adult Medicaid threshold without converting the household's MAGI to its income-to-FPL level or identifying a qualifying category. The correct computation places MAGI at 3.71 times FPL and assigns medicaid_category NONE, so the head is not eligible." us,scenario_036,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were provided and that payroll tax therefore equals zero, but submitted $3,715 instead. That amount applies a payroll-tax-style rate to the $59,220 taxable private pension, which is not employee compensation subject to payroll tax." us,scenario_036,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted the $1,000 premium twice (as both 'health insurance premiums' and 'other premiums') to reach $2,100 of medical costs, and took a $915.60 deduction that doesn't exist because qualifying costs are below the $1,184.40 2% floor. It then submitted $187, which contradicts its own computed $1,674. The correct tax on $58,220 is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a federal-style $6,500 standard deduction that NJ doesn't have, never applied the $1,000 NJ exemption, and stretched the 3.5% bracket to $60,000 when that band covers only $35,000–$40,000, with 5.525% above it. It then abandoned its own $1,417.50 for an unsupported $3,185. Correctly, $58,220 of taxable income yields $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It correctly said the pension exclusion doesn't apply at age 56, but then guessed about $1,000 instead of running the NJ brackets. Running $59,220 − $1,000 exemption = $58,220 through the 1.4%/1.75%/3.5%/5.525% schedule gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It first reached the correct $1,724.15, then subtracted an $8,405 medical deduction built from $9,589 that included the $7,389 employer-sponsored premiums and counted the $1,000 premium twice. The actual qualifying out-of-pocket expenses don't exceed the 2% floor of $1,184.40, so no deduction applies and the tax stays $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It took an $8,405 medical deduction that included the $7,389 ESI premiums and counted the $1,000 premium twice, which gave $1,260. It then made a further unsupported cut to $613 based on a supposed 'ESI premium treatment'. Qualifying expenses are below the 2% floor, so taxable income is $58,220 and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added the $7,389 employer-sponsored premiums and both $1,000 premium entries into a $9,589 medical total and deducted $8,404.60 above the 2% floor. ESI premiums aren't deductible and the $1,000 premium is a single amount, so no medical deduction exists. Taxable income is $58,220, not $49,815.40, and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It applied NJ's retirement-income exclusion to a 56-year-old who isn't disabled, although the exclusion requires age 62+ or disability. It then made up a 'phase-out remainder' to land on $1,591. The full pension is taxable, and $58,220 after the exemption produces $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It built $2,200 of medical costs by counting the $1,000 premium twice and including OTC expenses, and took a $1,016 deduction. Qualifying expenses don't exceed the $1,184.40 2% floor, so taxable income is $58,220, not $57,204, and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its bracket rates were right, but it applied them to the full $59,220 and left out the $1,000 NJ personal exemption. Taxing $58,220 instead removes $55.25 and gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It applied the $75,000 pension exclusion based on the income limit alone and ignored that the exclusion requires age 62+ or disability. At 56, the full $59,220 is taxable, which gives $1,724.16 after the $1,000 exemption." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It counted $2,100 of medical costs by double-counting the $1,000 premium and took a $915.60 deduction above the 2% floor. Qualifying expenses fall short of the $1,184.40 floor, so the only subtraction is the $1,000 exemption and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It correctly rejected the pension exclusion, but then deducted $8,404.60 of medical expenses from a $9,589 total that included the $7,389 employer-sponsored premiums and counted the $1,000 premium twice. No medical deduction clears the 2% floor, so taxable income is $58,220 and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. $1,113 implies roughly $10,000 of extra NJ deductions or exclusions on top of the $1,000 exemption, and none exist here. The correct path is $59,220 − $1,000 = $58,220, which the NJ brackets turn into $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"It excluded the whole pension under NJ's $75,000 exclusion while ignoring the age-62-or-disabled requirement. The 56-year-old head doesn't qualify, so $58,220 is taxable and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,age_disability,False,"It taxed only the part of the pension 'above the NJ pension exclusion', which means it gave a 56-year-old an exclusion reserved for people 62+ or disabled. With no exclusion, $58,220 is taxable and the tax is $1,724.16, not $1,057." -us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its taxable income of $58,220 was correct, but it used 2% and 2.5% for the first two brackets, while NJ's single schedule is 1.4% on the first $20,000 and 1.75% on $20,000–$35,000. With the correct rates the tax is $1,724.16, not $1,956.66." -us,scenario_036,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 NJ standard deduction (NJ has none), taxed the rest at a flat 5.525% instead of the graduated brackets, and subtracted a property-tax credit computed from the mortgage balance even though no property tax is listed. The correct base is $58,220 after the $1,000 exemption, taxed at $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation, and $2,860 is about $1,136 above the graduated result. That points to the 5.525% rate (or higher) being applied to most of the income without the lower 1.4%/1.75%/3.5% bands. The correct tax on $58,220 is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It subtracted unspecified 'standard credits' that NJ doesn't offer this filer, landing at $600. Nothing reduces the bracket tax: $58,220 after the exemption gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $7,304.60 of medical expenses from an $8,489 base that included the $7,389 employer-sponsored premiums, which aren't deductible out-of-pocket costs. The remaining qualifying expenses are below the 2% floor, so taxable income is $58,220, not $50,915.40, and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It treated the private pension as fully excluded because income was under the threshold, skipping the requirement that the filer be 62+ or disabled. At 56, the head's $59,220 is fully taxable, which gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took a medical-expense deduction of roughly $930 above the 2% floor. That only works by counting the $1,000 premium twice; qualifying expenses are below the $1,184.40 floor. Without the deduction, taxable income is $58,220 and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It called the head an 'eligible' filer for NJ's pension exclusion, but the exclusion requires age 62+ or disability and the head is 56. The pension is fully taxable, and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no computation. $1,125 implies roughly $10,800 of 'deductions and credits' that don't exist here, matching the incorrect medical-deduction shortcut. The only subtraction is the $1,000 exemption, so the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted $8,405 of medical expenses using $9,589 that included the ESI premiums and the double-counted $1,000. It then used a made-up schedule ($350 + 2.75% over $20,000) instead of NJ's 1.4%/1.75%/3.5%/5.525% brackets. Correctly, $58,220 of taxable income gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It deducted $8,405 of medical expenses built from $9,589 that included the $7,389 ESI premiums and the $1,000 premium counted twice. It also miscomputed the bracket tax on its own $49,815 base ($1,008.54 instead of $1,259.80). With no medical deduction, $58,220 is taxable and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is nothing substantive to evaluate." -us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It admitted the NJ pension exclusion is for residents 62+, then applied an invented $100,000 exclusion to the 56-year-old head anyway. No exclusion applies, so $58,220 is taxable and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted $2,200 of medical expenses by double-counting the $1,000 premium and including OTC costs, and deducted $1,015.60. Qualifying expenses don't exceed the $1,184.40 2% floor, so taxable income is $58,220, not $57,204.40, and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It explicitly computed the correct $1,724.16 on $58,220 of taxable income, then dropped it for an unsupported 'withholding-style flat estimation' of $2,283.06. That final figure has no basis in the NJ rate schedule." -us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a $16,100 standard deduction, which is a federal amount and doesn't exist in NJ, instead of the $1,000 exemption. It also used a wrong 5.5% rate and then added a 'property tax deduction' to the tax. The correct base is $58,220, which gives $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted the $1,000 'excluding Part B' premium line on top of the $1,000 other-premium line. That gave $2,100 of medical costs and a $916 deduction above the 2% floor, so it computed $1,673.55. It then submitted 187, which its own work doesn't support. Countable expenses are only $1,100, which is under the $1,184.40 floor, so taxable income is $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied a federal-style standard deduction, which NJ doesn't have. It also taxed the $35,000-$60,000 range at 3.5%, when the single 5.525% bracket starts at $40,000. It then submitted an unsupported $3,185. The correct calculation is $59,220 minus the $1,000 exemption, which gives $58,220 of taxable income and $1,724.16 of tax." +us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It correctly found the pension fully taxable at age 56 but never ran the brackets, and its $1,000 figure is a guess. Applying 1.4%/1.75%/3.5%/5.525% to $58,220 ($59,220 minus the $1,000 exemption) gives $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It reached the correct $1,724.15 and then dropped it to subtract an $8,405 medical deduction. That deduction is built from $9,589 that includes the $7,389 of ESI premiums, the duplicated $1,000 premium line, and OTC spending. Only $1,100 counts, which is under the 2% floor, so no medical deduction applies." +us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It took an $8,405 medical deduction from $9,589 of expenses that included the ESI premiums, the duplicated premium line, and OTC costs. It then cut the result further to $613 based on an invented ESI-premium treatment. No medical deduction applies because countable costs of $1,100 fall below the $1,184.40 floor, so taxable income stays at $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated all $9,589 of listed health costs as NJ medical expenses, including the $7,389 of ESI premiums, both premium lines, and OTC. That produced an $8,404.60 deduction and $49,815.40 of taxable income. Countable expenses are $1,100, which is below the 2% floor, so taxable income is $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It applied NJ's retirement income exclusion to a 56-year-old, but the exclusion requires age 62 or disability. It then submitted an unexplained $1,591 'phase-out remainder'. The full $59,220 is taxable, and after the $1,000 exemption the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It counted $2,200 of medical costs by adding both $1,000 premium lines plus OTC spending, and took a $1,016 deduction above the 2% floor. Countable expenses are $1,000 of premiums plus $100 of other medical, or $1,100. That doesn't reach the $1,184.40 floor, so taxable income is $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It ran the single brackets on the full $59,220 and left out the $1,000 single personal exemption. That overstated the 5.525% slice by $1,000, giving $1,779 instead of $1,724.16 on $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It applied the NJ pension exclusion to the whole $59,220 even though the head is 56 and not disabled, and the exclusion requires age 62 or disability. The pension is fully taxable, and the tax on $58,220 after the exemption is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It counted $2,100 of medical costs by adding the 'excluding Part B' premium line on top of the identical other-premium line, which gave a $915.60 deduction. The premium counts once, so countable expenses are $1,100. That is below the $1,184.40 floor, so no medical deduction applies and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It deducted $8,404.60 of medical expenses based on $9,589 that included the ESI premiums, both premium lines, and OTC costs. Countable expenses of $1,100 don't exceed the 2% floor. Taxable income is $59,220 − $1,000 = $58,220, which gives $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $1,113 corresponds to taxable income of roughly $47,000, which means an unexplained deduction of about $11,000. The only subtraction is the $1,000 exemption, so taxable income is $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"It applied the NJ pension exclusion based only on the income limit and ignored that the head must be 62 or older or disabled. At 56, the full $59,220 is taxable, which gives $1,724.16 after the $1,000 exemption." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,age_disability,False,"It treated part of the pension as excluded, but the NJ pension exclusion isn't available below age 62 without a disability. With the full $59,220 taxable and only the $1,000 exemption subtracted, the tax is $1,724.16, not $1,057." +us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly found taxable income of $58,220 but used non-existent rates of 2% and 2.5% for the first two single brackets. The actual rates are 1.4% on the first $20,000 and 1.75% on $20,000-$35,000. With those rates, the tax is $1,724.16, not $1,956.66." +us,scenario_036,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 NJ standard deduction and taxed everything above it at a flat 5.525%. It also subtracted a property-tax credit derived from the mortgage balance, but no property taxes were listed. NJ has no standard deduction, so the graduated brackets apply to $58,220 and give $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation, and $2,860 is far above what NJ's single brackets produce at this income. Applying 1.4%/1.75%/3.5%/5.525% to $58,220 after the $1,000 exemption gives $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It cut the tax to $600 using unspecified 'standard credits', but no NJ nonrefundable credits apply to this household. The tax on $58,220 of taxable income is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It included the $7,389 of ESI premiums in $8,489 of 'qualifying' medical costs and took a $7,304.60 deduction. ESI premiums aren't out-of-pocket deductible expenses here. The countable $1,100 falls below the 2% floor, so taxable income is $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It fully excluded the private pension under NJ's pension exclusion, which requires the taxpayer to be 62 or older or disabled. The 56-year-old head doesn't qualify, so $58,220 is taxable after the exemption and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took a medical deduction of roughly $900 by treating the reported premium lines as separate out-of-pocket costs above the 2% threshold. The premium counts once, and $1,000 of premiums plus $100 of other medical costs is under the $1,184.40 floor. That leaves taxable income of $58,220 and tax of $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It called the head an 'eligible' filer for the NJ pension exclusion, but the exclusion requires age 62 or disability and the head is 56. The whole pension is taxable, so the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $1,125 corresponds to roughly $47,000 of taxable income, which means unexplained deductions of about $11,000. The only subtraction is the $1,000 exemption, so taxable income is $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted $8,405 of medical expenses based on $9,589 that included the ESI premiums, both premium lines, and OTC costs. It then used an invented '350 plus 2.75%' rate formula. No medical deduction applies, and the 1.4%/1.75%/3.5%/5.525% brackets on $58,220 give $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction built from ESI premiums, duplicated premiums, and OTC costs. It also miscalculated the brackets on its own $49,815 figure, which gives $1,259.80, not $1,008.54. With no medical deduction, taxable income is $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The answer had no parseable value and no explanation, so there was no figure to compare with the $1,724.16 bracket tax on $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It acknowledged that the pension exclusion is for taxpayers 62 and older, then applied it to the 56-year-old head anyway. The pension is fully taxable, which gives $1,724.16 on $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It took a medical deduction of about $8,405 by counting the ESI premiums, both premium lines, and OTC costs against the 2% floor. Countable costs of $1,100 don't exceed $1,184.40, so taxable income is $58,220, not $49,815, and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted $2,200 of medical costs by adding the 'excluding Part B' premium line on top of the identical other-premium line and including OTC spending, then deducted $1,015.60. Countable expenses are $1,100, which is below the floor. Taxable income is therefore $58,220 and the tax is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It correctly computed $1,724.16 on $58,220 after the $1,000 exemption. It then discarded that result for an unsupported $2,283.06 based on a 'withholding-style flat estimation'." +us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a $16,100 'standard deduction' that NJ doesn't have, used a 5.5% rate in place of 5.525%, and added an invented property-tax item. The only subtraction is the $1,000 exemption, and the single brackets on $58,220 give $1,724.16." us,scenario_036,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It stated that the $9,629 FLSA overtime premium is not deductible, so it missed the 2026 OBBBA qualified overtime deduction. It also subtracted the $5,789 ESI premiums from wages and used the 2024 $14,600 standard deduction and $11,600 10% bracket instead of $16,100 and $12,400. It computed $2,308, then submitted $1,646, a figure its own arithmetic does not support." us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It counted the overtime premium twice by adding it on top of wages that already include it ($51,572). It then claimed tax is $0. In fact, after the $16,100 standard deduction and the $9,629 overtime deduction, $15,810 of taxable income remains, which produces $1,649.25 at the 10%/12% rates." @@ -3397,47 +3448,48 @@ us,scenario_038,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibili us,scenario_038,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly placed a six-year-old within the modeled WIC child age category. Because eligibility ends at the fifth birthday, Child 2 fails the categorical age test." us,scenario_038,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable output for the requested variable and therefore failed the required submission contract. us,scenario_038,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly derived zero taxable income, zero tax before credits, and zero nonrefundable credits used, but its submitted value of -$1,644 contradicts that derivation. It inserted an unsupported negative amount even though nonrefundable credits cannot reduce liability below zero and refundable credits are excluded from this output." -us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It computed the ACTC correctly at $2,909, then arbitrarily cut the EITC from its $7,427 estimate to an unsupported $7,239. The 2026 two-child plateau maximum is $7,316, which gives a total of $10,225." -us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It invented a $1,656 EITC phase-out even though earned income of about $21,896 is below the joint phase-out start of about $31,000, so the full $7,316 maximum applies. It also gave the full $3,400 ACTC cap without applying the 15%-of-earnings-above-$2,500 limit, which gives $2,909." -us,scenario_038,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It assumed income tax fully absorbed the CTC, but income tax before credits is $0, so $2,909 is refundable through the 15% earned-income formula. It also gave far too low an EITC: earned income of $21,896 is on the plateau, so the EITC is the $7,316 maximum." -us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an EITC of $7,150, below both the true 2026 maximum of $7,316 and its own stated $7,250 plateau. It then calculated the ACTC as $2,909 but entered an unexplained $2,383 in the total." -us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It put the EITC at $5,500, as if earnings were still in the phase-in range, but $21,896 is past the roughly $18,290 phase-in end, so the EITC is the full $7,316. It also cut the ACTC to an arbitrary $1,527 even though the 15% formula gives $2,909, below the $3,400 cap." -us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It underestimated the 2026 two-child EITC maximum as about $7,000 instead of $7,316. It also based the ACTC on gross earnings ($3,074) without subtracting half of the SE tax, and its $9,199 total doesn't match either component." -us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It calculated the ACTC correctly at $2,909, but used $6,604 as the EITC maximum. That is the 2023 two-child figure, not the 2026 maximum of $7,316, which understates the total by $712." -us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It estimated the EITC maximum at about $6,900 instead of $7,316. It then cut its own $9,796 sum to $8,500 with an unjustified 'phase-in' adjustment, even though earned income of $21,896 is on the plateau." -us,scenario_038,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as $7,430 instead of $7,316. It also measured ACTC earnings as wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax ($21,896), which gave $2,895.80 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child CTC with a $3,000 earnings floor, which capped the ACTC at $2,000. Under the 2026 law, the CTC is $2,200 per child, the refundable cap is $1,700 per child, and the floor is $2,500, which gives $2,909. It also overstated the EITC at $7,354 instead of $7,316." -us,scenario_038,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It capped the ACTC at $2,000, using $1,000 per child and a $3,000 floor. The 2026 rules have a $1,700-per-child refundable cap and a $2,500 floor, which gives $2,909. It also used $7,344 instead of the $7,316 EITC maximum." -us,scenario_038,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used $7,350 for the EITC maximum instead of the 2026 figure of $7,316. It also based the ACTC on wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax ($21,896), which gave $2,895.80 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated the refundable CTC as $1,000 per child with a $3,000 floor, totaling $2,000. Under the 2026 rules, the refundable CTC is up to $1,700 per child, phased in at 15% above $2,500, which gives $2,909. It also understated the EITC at $7,300 instead of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It treated the full $4,000 CTC as refundable, ignoring the $1,700-per-child refundable cap and the 15%-of-earnings-above-$2,500 limit, which gives $2,909. It also put the EITC at $4,100, even though earned income of $21,896 is on the plateau, where the credit is the full $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $2,000 CTC, reflecting the pre-TCJA $1,000 per child. The 2026 refundable CTC formula gives $2,909: 15% of $19,396, below the $3,400 cap. It also put the EITC at $7,300 instead of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It capped the ACTC at $1,000 per child ($2,000 total) under the pre-TCJA rules. In 2026 the refundable cap is $1,700 per child, so the 15% formula's $2,909 applies. It also used $7,250 instead of the $7,316 EITC maximum." -us,scenario_038,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $7,466 total is about $2,759 below the correct figure. That matches dropping most of one credit, such as using the EITC plus only a token ACTC. The correct components are the $7,316 plateau-maximum EITC and a $2,909 ACTC (15% of the $19,396 of earnings above $2,500)." -us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $9,753.44 total is about $472 short. That matches an understated EITC maximum or ACTC earnings base. The correct components are the $7,316 two-child EITC maximum, since earned income of $21,896 is on the plateau, and a $2,909.35 ACTC." -us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $10,185 total is $40 short of $7,316 EITC plus $2,909.35 ACTC. That matches using a projected EITC maximum near $7,276 instead of the published 2026 figure of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It set the ACTC at $2,000, using the pre-TCJA $1,000 per child, instead of applying the 2026 cap of $1,700 per child to the 15% formula, which gives $2,909. It also used $7,326 instead of the $7,316 EITC maximum." -us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no parseable value for federal_refundable_credits, so there was no answer to score against $10,225.35." -us,scenario_038,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It measured earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax ($21,896), which understated the ACTC at $2,896. It also rounded the EITC to $7,300 instead of the 2026 maximum of $7,316." -us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $8,724 total is about $1,501 short. That matches an EITC well below the plateau maximum or a pre-TCJA $2,000 ACTC. The correct EITC is the $7,316 maximum, since earned income of $21,896 is on the plateau, and the ACTC is $2,909 from 15% of earnings above $2,500." -us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It reported no refundable credits, even though the joint filers have $21,896 of earned income and two qualifying children. That qualifies them for the full $7,316 EITC and a $2,909 refundable CTC." -us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Its EITC of $7,316 is correct, but it gave a flat $1,800 per child ($3,600) of refundable CTC. It ignored the earned-income phase-in, which limits the refundable CTC to 15% × ($21,896 − $2,500) = $2,909, below the $1,700-per-child cap." -us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $8,231, the 2026 EITC maximum for three or more children, instead of the $7,316 two-child maximum. It also based the ACTC on gross earnings of $22,993 without subtracting half the SE tax, which gave $3,073.95 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its EITC of $7,316 is correct, but its $2,731.32 ACTC implies earned income of $20,709. That matches applying the 92.35% factor to SE income and then also subtracting half the SE tax. Net SE earnings are $15,525 minus $1,097 only, so earned income is $21,896 and the ACTC is $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Its EITC of $7,316 is correct, but it gave the full $1,700-per-child refundable cap ($3,400) without the earned-income limit. The refundable CTC is limited to 15% × ($21,896 − $2,500) = $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its $2,909.46 ACTC is essentially correct, but it used $7,427 as the two-child EITC maximum instead of the published 2026 figure of $7,316, which overstates the total by $111." -us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It wrongly treated the 'do not infer unlisted facts' instruction as a bar on computing credits and returned $0. The listed wages, SE income and two children directly produce a $7,316 EITC and a $2,909 refundable CTC." -us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA lapsed, so it used a $1,000 CTC per child and an ACTC of $2,000. For 2026, the refundable cap is $1,700 per child, so the 15% formula's $2,909 applies. It also uprated the EITC maximum to $7,312 instead of the published $7,316." -us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child ACTC with a $3,000 floor ($2,000 total) instead of the 2026 cap of $1,700 per child and the $2,500 floor, which give $2,909. It also projected a $7,330 EITC maximum instead of $7,316." -us,scenario_038,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed the CTC reverted to $1,000 per child after 2025, giving a $2,000 refundable CTC. The 2026 law keeps a $2,200 CTC with a $1,700 refundable cap, so the 15% formula gives $2,909. It also used $7,332 instead of the $7,316 EITC maximum." -us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied post-TCJA reversion rules ($1,000 per child, $3,000 floor) for an ACTC of $2,000, instead of the 2026 cap of $1,700 per child and the $2,500 floor, which give $2,909. It also projected the EITC at $7,338 instead of $7,316." -us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as $7,422 instead of $7,316. Its ACTC of $2,879 is below the correct $2,909.35, which is 15% × ($21,896 − $2,500)." -us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no parseable value for federal_refundable_credits, so there was no answer to score against $10,225.35." -us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $7,316 is correct, but it based the ACTC on gross earnings of $22,993. Earned income for the CTC phase-in counts SE income net of half the SE tax ($21,896), so the ACTC is $2,909.35, not $3,073.95." -us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It understated the EITC maximum at $6,500 instead of the 2026 plateau amount of $7,316. It also gave the full $3,400 ACTC cap without the 15%-above-$2,500 limit, which caps it at $2,909." -us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its $2,910 ACTC is correct, but it used $7,370 as the 2026 two-child EITC maximum instead of $7,316, which overstates the total by about $55." -us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It wrongly applied a $21,370 joint EITC phase-out start; in 2026 the phase-out starts around $31,000, so $21,896 of earnings is on the plateau and earns $7,316. It then abandoned its own ACTC calculation of $2,909 and submitted an unsupported EITC-only figure of $6,868.75." -us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed earned income of $22,993 was past the EITC phase-out and gave only $46. For joint filers with two children, the 2026 phase-out starts around $31,000, so the full $7,316 applies. It also rounded the ACTC to $3,000 instead of $2,909.35, which uses earnings net of half the SE tax." +us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly computed the refundable CTC as $2,909 and placed earned income in the EITC plateau. It then swapped its $7,428 EITC estimate for an invented $7,239 maximum. The 2026 two-child EITC maximum is $7,316, so the total is $10,225, not $10,148." +us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It capped the EITC at a 40% phase-in on $15,628 ($6,251) and subtracted a $1,656 phase-out that does not apply. Earned income of about $21,896 is in the plateau below the MFJ phase-out start, so the full $7,316 applies. It also took the $3,400 CTC cap without the 15%-over-$2,500 limit ($2,909), and its own components add to $7,995, not the $3,995 it submitted." +us,scenario_038,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It assumed tax liability fully absorbed the CTC and gave only a small $1,500 EITC. Income tax before credits is $0, so the refundable CTC is the full 15% × ($21,896 − $2,500) = $2,909. Earned income is in the plateau, so the EITC is the full $7,316." +us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an EITC of about $7,150 instead of the 2026 two-child maximum of $7,316. It correctly found that the ACTC formula gives $2,909 under the $3,400 cap, then entered an unexplained $2,383 in its total." +us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It put the household below the EITC plateau ($5,500), but earned income of about $21,896 is past the phase-in end, so the full $7,316 applies. It computed a refundable CTC of $3,074 and then inexplicably used $1,527. The correct ACTC is 15% × ($21,896 − $2,500) = $2,909." +us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used an EITC of about $7,000 instead of the 2026 maximum of $7,316. It computed the ACTC on gross SE income ($3,074) instead of earned income net of the deductible half of SE tax ($2,909). Its $9,199 total does not even match its own components." +us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed the refundable CTC correctly at $2,909. For the EITC it used $6,604, an outdated two-child maximum it took to be 2025's, as the 2026 value. The 2026 two-child maximum is $7,316, which accounts for the $712 shortfall." +us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It estimated the EITC at about $6,900, below the $7,316 plateau maximum, and measured earned income as 92.35% of SE income instead of SE income minus half the SE tax. It then dropped its own $9,796 sum to an arbitrary $8,500 with no rule supporting the cut." +us,scenario_038,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an EITC maximum of $7,430 instead of the 2026 two-child value of $7,316. It also took earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus the deductible half of SE tax ($21,896), so its ACTC was $2,895.80 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-2018 $1,000-per-child CTC with a $3,000 earnings threshold, capping the ACTC at $2,000. The 2026 CTC refunds up to $1,700 per child and uses the $2,500 threshold, giving $2,909. Its $7,354 EITC also exceeds the $7,316 maximum." +us,scenario_038,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It treated the 2026 CTC as having reverted to $1,000 per child with a $3,000 threshold, giving a $2,000 ACTC. The 2026 law allows $1,700 refundable per child with 15% phase-in above $2,500, giving $2,909. Its $7,344 EITC also exceeds the $7,316 two-child maximum." +us,scenario_038,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used an EITC maximum of $7,350 instead of $7,316. It computed the ACTC on earned income of $21,805 (92.35% of SE income) instead of $21,896 (SE income minus the deductible half of SE tax), so the ACTC came out at $2,895.80 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the expired $1,000-per-child refundable CTC with a $3,000 threshold, a $2,000 total. The 2026 rules refund up to $1,700 per child at 15% above $2,500, which gives $2,909. It also understated the EITC at $7,300 instead of $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It treated the whole $4,000 CTC as refundable, ignoring the $1,700-per-child refundable cap and the 15%-over-$2,500 limit, which together give $2,909. Its $4,100 EITC also misses that earned income of about $21,896 is in the plateau, where the full $7,316 applies." +us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $2,000 CTC, the expired $1,000-per-child figure, instead of the 2026 refundable CTC of 15% × ($21,896 − $2,500) = $2,909 under the $3,400 cap. It also estimated the EITC at $7,300 instead of the $7,316 maximum." +us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the expired $1,000-per-child ACTC ($2,000) instead of the 2026 refundable CTC of $2,909, which is capped at $1,700 per child and phases in at 15% above $2,500. It also understated the EITC at $7,250 instead of $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"The correct total is the $7,316 EITC plus a $2,909 refundable CTC. Its $7,466 is only $150 above the EITC alone, which matches dropping nearly all of the refundable CTC, $2,909 for a family with $21,896 of earned income and no income tax liability." +us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Its $9,753.44 is $472 short of the $7,316 maximum EITC plus the $2,909.35 refundable CTC. That matches an EITC reduced to about $6,844 even though earned income of about $21,896 is in the plateau below the MFJ phase-out start." +us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $10,185 is $40 below the correct $7,316 EITC plus $2,909.35 refundable CTC. That matches using an EITC maximum of about $7,276 instead of the 2026 two-child value of $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $2,000 refundable CTC ($1,000 per child), as if the 2017 tax-law (TCJA) changes had expired, instead of the 2026 rules, which give 15% × ($21,896 − $2,500) = $2,909 under a $1,700-per-child cap. It also overstated the EITC at $7,326 instead of $7,316." +us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no usable value for federal_refundable_credits, so nothing could be scored against the $7,316 EITC plus $2,909.35 refundable CTC." +us,scenario_038,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used an EITC of $7,300 instead of the 2026 two-child maximum of $7,316. It measured earned income as 92.35% of SE income ($21,805) instead of SE income minus the deductible half of SE tax ($21,896), so the ACTC came out at $2,896 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $8,724 is about $1,500 short of the correct $7,316 EITC plus $2,909 refundable CTC. That matches an EITC below the plateau maximum or a refundable CTC under the 15% × ($21,896 − $2,500) amount, even though both credits are fully available at this income." +us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It concluded that no refundable credits arise. A married couple with two children under 17 and about $21,896 of earned income gets the full $7,316 EITC and a $2,909 refundable CTC, because income tax liability is $0." +us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Its $7,316 EITC is correct. For the refundable CTC it used $1,800 per child ($3,600), which is the wrong cap ($1,700 per child), and it skipped the 15%-over-$2,500 earned income limit, which caps the credit at $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used an $8,231 EITC, near the three-or-more-child maximum, instead of the $7,316 two-child maximum. It also computed the ACTC on gross wages plus SE income ($22,993) instead of earned income net of the deductible half of SE tax ($21,896), which overstates it at $3,073.95 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $7,316 EITC is correct. Its $2,731.32 ACTC implies earned income of about $20,709, meaning it cut SE income twice: once by the 92.35% factor and again by half the SE tax. The correct earned income is $15,525 − $1,097 + $7,468 ≈ $21,896, which gives $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Its $7,316 EITC is correct. It granted the full $1,700-per-child refundable CTC ($3,400) without the 15%-over-$2,500 earned income limit, which binds at 15% × ($21,896 − $2,500) = $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of about $2,909 is correct. It used an EITC maximum of $7,427 instead of the 2026 two-child value of $7,316, which overstates the total by about $111." +us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It read the instruction not to infer unlisted facts as a bar on computing credits and returned $0. The listed wages, SE income, and two qualifying children directly produce the $7,316 EITC and the $2,909 refundable CTC." +us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It capped the ACTC at $2,000 under the expired $1,000-per-child rule. The 2026 refundable cap is $1,700 per child ($3,400), so the 15% × ($21,896 − $2,500) = $2,909 phase-in amount applies. Its uprated EITC of $7,312 is also slightly below the $7,316 maximum." +us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the CTC reverted to $1,000 per child with a $3,000 threshold, giving a $2,000 ACTC. Under 2026 law the refundable CTC is $2,909 (15% above $2,500, with a $1,700-per-child cap). It also used $7,330 instead of the $7,316 EITC maximum." +us,scenario_038,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a post-2025 reversion to a $1,000-per-child CTC, giving a $2,000 refundable amount. The 2026 CTC refunds up to $1,700 per child at 15% of earnings above $2,500, which is $2,909 here. It also used $7,332 instead of the $7,316 EITC maximum." +us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the reverted $1,000-per-child CTC with a $3,000 threshold ($2,000) instead of the 2026 refundable CTC of $2,909. It also used an EITC of $7,338 instead of $7,316, and treated gross SE income as earned income without subtracting half the SE tax." +us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the EITC at about $7,422 instead of the $7,316 two-child maximum. Its ACTC of $2,879 implies an earned income figure below the correct $21,896 (wages plus SE income minus half SE tax), which gives $2,909.35." +us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no usable value for federal_refundable_credits, so nothing could be scored against the $7,316 EITC plus $2,909.35 refundable CTC." +us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its $7,316 EITC is correct. It computed the ACTC on gross earnings of $22,993 instead of earned income net of the deductible half of SE tax ($21,896), which overstates the refundable CTC at $3,073.95 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It understated the EITC at about $6,500 instead of the $7,316 plateau maximum. It also granted the full $3,400 ACTC cap without the 15%-over-$2,500 earned income limit, which yields $2,909." +us,scenario_038,federal_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used an EITC of about $7,148, close to the 2025 two-child maximum, instead of the 2026 value of $7,316. It computed the ACTC on earned income of $21,805 (92.35% of SE income) instead of $21,896, so the ACTC came out at $2,896 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its $2,910 refundable CTC is correct. It overstated the EITC maximum at about $7,370 instead of the 2026 two-child value of $7,316, which inflates the total by about $55." +us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It at one point reached about $7,152 EITC plus $3,074 ACTC, then abandoned that and submitted $6,868.75. That figure leaves out the refundable CTC entirely and treats about $21,370 as the start of the EITC phase-out, a misremembered threshold. The correct values are the full $7,316 EITC (earned income is in the plateau) plus a $2,909 refundable CTC." +us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed earned income of $22,993 exceeds the EITC phase-out and gave a $46 EITC. The MFJ two-child phase-out begins well above about $21,896, so the full $7,316 applies. It also rounded the ACTC to $3,000 using gross earnings instead of 15% × ($21,896 − $2,500) = $2,909." us,scenario_038,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted nonqualification without applying either qualifying pathway. PolicyEngine derives categorical eligibility from the household's $7,286.94 annual SNAP benefit and independently derives income eligibility because the school-meal FPG ratio is 0.70, below 1.30." us,scenario_038,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly computed gross income below 130% FPG but then incorrectly imposed SNAP receipt or direct certification as an additional requirement and treated an unlisted SNAP input as proof of no SNAP participation. Income at or below 130% FPG independently qualifies the children for free meals, and PolicyEngine also computes $7,286.94 of SNAP under the assumed-take-up instruction, which supplies categorical eligibility and produces positive meal support." us,scenario_038,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -3464,45 +3516,46 @@ us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,Fals us,scenario_038,self_employment_tax,grok-4.3,llm_error,other,False,"It labeled its answer approximate and gave $2,192 instead of computing exactly. The exact figure is $15,525 × 0.9235 × 0.153 = $2,193.61." us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for self_employment_tax, so there was nothing to score. The correct result is $15,525 × 0.9235 × 0.153 = $2,193.61." us,scenario_038,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It correctly reached $2,193.61, then applied the deduction for half of SE tax ($1,096.81) a second time, iteratively reducing the SE tax base itself and ending at $2,054.08. Under IRC §164(f) that deduction reduces only AGI for income tax. The 92.35% factor already accounts for it, so the SE tax stays at $2,193.61." -us,scenario_038,snap,claude-fable-5,llm_error,other,False,"Its own working reached about $7,236 per year, but it submitted 4,200, a number its reasoning does not support. The working also used a $212 standard deduction and a $1,000 maximum allotment instead of the FY2026 values of $223 and $994, which give $601 a month." -us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It skipped the 20% earned income deduction and used a $194 standard deduction instead of $223. It also invented a $400 shelter deduction from the mortgage balance, even though no housing payment is listed. It then dropped its own $647/month result for an unexplained $84/month, far below the correct $601 ($994 − 30% × $1,310)." -us,scenario_038,snap,claude-haiku-5.5,llm_error,other,False,"It showed no calculation. Its $3,600 equals $300 a month, about half of the correct $601: $994 maximum allotment minus 30% of $1,310 net income, after the $223 standard deduction and the $383 earned income deduction." -us,scenario_038,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted a monthly standard deduction ($224) from annual income ($18,394), which inflated monthly net income to $1,514 instead of $1,310. It then dropped its own $540/month result and submitted an unsupported $759/month ($9,114/yr)." -us,scenario_038,snap,claude-opus-4.8,llm_error,other,False,"With its own inputs ($994 maximum, $217 standard deduction, 20% earned income deduction), net income is about $1,316 and the benefit is about $599 a month. It never did that arithmetic and asserted $654/month ($7,848/yr) instead. The correct figure with the $223 deduction is $601." -us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"It gave no parameters and asserted $564 a month. That is $37 below the correct $601, which comes from the $994 FY2026 maximum allotment minus 30% of $1,310 net income after the $223 standard deduction. Its figure reflects an understated allotment or overstated net income." -us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It deliberately used FY2025 parameters, a $975 maximum allotment and a $204 standard deduction, for a 2026 household. The FY2026 values are $994 and $223, which give net income of $1,310 and a $601 monthly benefit instead of its $576." -us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It cut self-employment income by 20% before counting it, then applied the 20% earned income deduction to the reduced amount. It also assumed dependent care and medical deductions, although there are no care costs and no elderly or disabled members, which understated net income at $900–1,000. Even with that net income, its $994-based arithmetic doesn't produce the $570–600 a month it reported." -us,scenario_038,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It understated gross income at $21,896 instead of $22,993 (wages plus self-employment income). It used an annual maximum allotment of $13,164, which is $1,097 a month instead of $994, and a $219 standard deduction instead of $223, which inflated the benefit to $8,697." -us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment and a $208 standard deduction instead of the FY2026 values of $994 and $223. That overstated the monthly benefit at $652.54 instead of $601." -us,scenario_038,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied FY2025 parameters, a $975 maximum allotment and a $204 standard deduction, instead of the FY2026 values of $994 and $223. That gives $576.34 a month instead of $601." -us,scenario_038,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $1,000 maximum allotment and a $213 standard deduction instead of $994 and $223, and it didn't round net income or the allotment to whole dollars. The result was $604.04 a month instead of $601." -us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It invented a $310/month 'dependent deduction'. SNAP has no per-dependent deduction, and the dependent care deduction requires actual care costs, which this household doesn't have. It also used the FY2024 $973 maximum allotment and a $198 standard deduction instead of $994 and $223." -us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It did no calculation. Its $9,000 ($750/month) overstates the benefit, which is the $994 maximum allotment minus 30% of $1,310 net income, or $601 a month." -us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It described the right formula but showed no numbers. Its $6,878 (about $573/month) understates the correct $601, which comes from the $994 FY2026 maximum allotment minus $393 (30% of $1,310 net income after the $223 standard deduction)." -us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $990 maximum allotment instead of $994. Its net income of $1,333 implies a standard deduction of about $200 instead of the FY2026 $223, so its benefit came out at $590 a month instead of $601." -us,scenario_038,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It said the household's net income disqualifies it, but net income of $1,310 is only 49% of the poverty guideline. The household passes every test and is categorically eligible through TANF non-cash (broad-based categorical) eligibility, so it receives $601 a month." -us,scenario_038,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly found the household eligible but gave $6,276 ($523/month) with no calculation. The correct benefit is the $994 maximum allotment minus 30% of $1,310 net income, or $601 a month." -us,scenario_038,snap,gemini-3.7-flash,llm_error,other,False,"It showed no calculation. Its $9,708 ($809/month) overstates the benefit, which is the $994 maximum allotment minus 30% of $1,310 net income, or $601 a month." -us,scenario_038,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It cited the right formula but no parameters. Its $580.50/month matches FY2025 values (around a $975 allotment) rather than the FY2026 $994 allotment and $223 standard deduction, which give $601." -us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $1,020 maximum allotment instead of the FY2026 $994 and a $220 standard deduction instead of $223. That overstated the benefit at $626.14 a month instead of $601." -us,scenario_038,snap,glm-5.3,llm_error,other,False,"It reached the correct net income of $1,310 but took 30% of it as $394 instead of $393 and used a $995 allotment. It then rounded to 'about $600/month' instead of computing $994 − $393 = $601, which is why it submitted $7,200 instead of $7,212." -us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It said earnings and assets rule out a benefit. In fact, gross income is 72% of the poverty guideline, assets are $500, and the household is categorically eligible through TANF non-cash eligibility, so it receives $994 − $393 = $601 a month." -us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 without applying the income tests. Gross income is 72% and net income 49% of the poverty guideline, and the household has categorical eligibility, so the benefit is $601 a month." -us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"It used a four-person maximum allotment of $879 instead of the FY2026 $994. Its net income of $1,316 implies a standard deduction of about $217 instead of $223, so it got $484 a month instead of $601." -us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $609.74/month implies a maximum allotment of about $1,003 or net income below $1,310, and it never rounded to whole dollars. The FY2026 $994 maximum minus $393 gives exactly $601." -us,scenario_038,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $595/month matches the $994 maximum allotment combined with the FY2025 $204 standard deduction (net about $1,329, 30% about $399). The FY2026 $223 standard deduction gives net income of $1,310 and a $601 benefit." -us,scenario_038,snap,grok-4.3,llm_error,missing_output,False,"It refused to estimate and entered $0, wrongly treating the absence of program parameters as a reason to report no benefit. The listed facts are enough to compute $601 a month ($994 maximum minus 30% of $1,310 net income)." -us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a projected $1,000 maximum allotment instead of $994 and a $225 standard deduction instead of $223, giving $608 a month. It then annualized that inconsistently as $7,280, while the correct figure is $601 × 12 = $7,212." -us,scenario_038,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction and $1,310 net income, but it used a $993 maximum allotment instead of the FY2026 $994. That made the benefit $600 a month instead of $601." -us,scenario_038,snap,grok-4.7,llm_error,thresholds_rates,False,"It used an $11,700 annual maximum allotment ($975/month, the FY2025 value) and a $217/month standard deduction ($2,604/yr) instead of the FY2026 values of $994 and $223. That understated the annual benefit at $6,963." -us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,985 annual standard deduction (about $332/month, far above $223) and a $12,337 annual maximum allotment (about $1,028/month instead of $994). Both errors push the benefit up, to $8,014." -us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"It used a $1,000 maximum allotment instead of $994. Its net income of $1,316 implies a standard deduction of about $217 instead of $223, which gave $605 a month instead of $601." -us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $208 standard deduction and a $1,020 maximum allotment instead of the FY2026 values of $223 and $994. That overstated the benefit at $622 a month instead of $601." -us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction but used a $997 maximum allotment instead of $994. It also didn't round net income to $1,310 or the allotment to whole dollars, so it got $604.04 a month instead of $601." -us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment and about a $204 standard deduction (net $1,329) instead of the FY2026 values of $994 and $223. That overstated the benefit at about $651 a month instead of $601." -us,scenario_038,snap,ox-alpha,llm_error,thresholds_rates,False,"It correctly used the $994 maximum allotment but used a $207 standard deduction instead of the FY2026 four-person value of $223. That made net income $1,326 instead of $1,310 and the benefit about $596 instead of $601." -us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"With a $198 standard deduction and a $973 maximum allotment, it got $572.54 a month. It then invented a shelter deduction to raise that to $879, even though the household has no rent, mortgage payment or utility costs." -us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It set net income to $0 by assuming a shelter deduction, even though no housing costs are listed. It also skipped the 30% contribution on $1,310 net income and used a $964 maximum allotment instead of $994." +us,scenario_038,snap,claude-fable-5,llm_error,other,False,"Its own derivation used a $212 standard deduction and a $1,000 maximum allotment and came to about $7,236. It then submitted $4,200, a number that does not follow from any step it showed. The correct inputs ($223 standard deduction, $994 maximum) give $601 a month." +us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the 20% earned income deduction and used a $194 standard deduction instead of $223. It also invented a $400 shelter deduction from the mortgage balance, although no mortgage payment, property tax or utility costs were given. It then dropped that result and asserted $84 a month with no derivation, far below the correct $994 − $393 = $601." +us,scenario_038,snap,claude-haiku-5.5,llm_error,other,False,"It gave a round $300 a month ($3,600 a year) with no calculation. The correct figure is $994 minus 30% of $1,310 net income (after the $383 earned income deduction and $223 standard deduction), which is $601 a month. Even with no deductions at all the benefit would be about $419 a month, so $300 follows from no valid calculation." +us,scenario_038,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted a monthly standard deduction (about $224) from annual net earned income of $18,394, so the deduction was effectively only about $19 a month, giving net income of $1,514 instead of $1,310. It then raised the benefit to $759 a month with no basis, rather than computing $994 − $393 = $601." +us,scenario_038,snap,claude-opus-4.8,llm_error,other,False,"Its own inputs ($1,916 gross, $383 earned income deduction, $217 standard deduction, $994 maximum) give about $599 a month. It submitted $654 a month ($7,848), which does not follow from those inputs. It also used $217 instead of the FY2026 standard deduction of $223." +us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"It estimated $564 a month without stating its parameters. The correct calculation is net income of $1,310 (after the $383 earned income deduction and $223 standard deduction) against the $994 FY2026 maximum, which gives $601. Its figure matches an outdated lower maximum allotment and/or smaller deductions." +us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used FY2025 parameters: a $975 maximum allotment and a $204 standard deduction. The FY2026 values are $994 and $223, so its net income was $1,329 instead of $1,310 and its benefit $576 a month instead of $601." +us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It reduced self-employment income by 20% before computing gross income and then applied the 20% earned income deduction again. It also added dependent-care and medical deductions, even though there are no dependent-care costs and no elderly or disabled members. Finally, its $6,800 does not follow from its own stated net-income range." +us,scenario_038,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It cut gross income to $21,896 by subtracting the deductible half of self-employment tax, which SNAP does not allow. It also used an inflated maximum allotment of $13,164 a year ($1,097 a month) instead of $994 a month." +us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment instead of the FY2026 four-person maximum of $994, and a $208 standard deduction instead of $223. This overstated the benefit at $652.54 a month, and it also skipped whole-dollar rounding." +us,scenario_038,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used FY2025 parameters: a $975 maximum allotment and a $204 standard deduction, instead of the FY2026 $994 and $223. That gave $576.34 a month instead of $601, and it did not round net income or the benefit to whole dollars." +us,scenario_038,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $213 standard deduction instead of $223 and a $1,000 maximum allotment instead of $994. It also did not round net income to $1,310 or the benefit to whole dollars, so it got $604.04 a month instead of $601." +us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It invented a $310 monthly 'dependent deduction', although no dependent-care costs were given and SNAP has no per-dependent deduction. It also used the 2024 $973 maximum allotment and a $198 standard deduction instead of the FY2026 $994 and $223." +us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave a round $750 a month ($9,000 a year) with no calculation. The correct calculation is $994 − 30% × $1,310 = $601 a month. Its figure implies net income of only about $813, which no allowable deductions in this case produce." +us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It described the right formula but reached $573 a month. That matches outdated FY2025 parameters (about a $975 maximum and a $204 standard deduction), not the FY2026 $994 maximum and $223 standard deduction that give $601." +us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a standard deduction of about $200 (net income $1,333) instead of $223 (net income $1,310), and a $990 maximum allotment instead of $994. That gave $590 a month instead of $601." +us,scenario_038,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It wrongly concluded the benefit is zero. Gross income of $1,916 a month is about 72% of the poverty guideline, net income of $1,310 is about 49%, and the household is categorically eligible through BBCE (non-cash TANF). The benefit is $994 − $393 = $601 a month." +us,scenario_038,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It correctly found the household eligible but gave $523 a month. Against the $994 maximum, that implies net income of about $1,570, which matches applying only one of the two deductions instead of both the $383 earned income deduction and the $223 standard deduction (net $1,310, benefit $601)." +us,scenario_038,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave $809 a month with no derivation. Against the $994 maximum, that implies net income of only about $617, which requires a shelter or other deduction the facts do not support. With only the $383 earned income deduction and the $223 standard deduction, net income is $1,310 and the benefit $601." +us,scenario_038,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It named the correct deductions but reached $580.50 a month. That matches outdated FY2025 maximum-allotment and standard-deduction values instead of FY2026's $994 and $223, which give $601." +us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $1,020 maximum allotment instead of the FY2026 four-person value of $994, and a $220 standard deduction instead of $223. It also skipped whole-dollar rounding, which gave $626.14 a month instead of $601." +us,scenario_038,snap,glm-5.3,llm_error,other,False,"It reached net income of about $1,310 but used a $995 maximum and a $394 contribution, then rounded to 'about $600 a month'. The exact figures are $994 − $393 = $601, or $7,212 a year." +us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It wrongly treated the household as getting no benefit. Gross income ($1,916 a month) and net income ($1,310) are far below the 130% and 100% poverty limits, assets are $500, and the household is categorically eligible through BBCE. The benefit is $994 − $393 = $601 a month." +us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed no positive benefit without applying the SNAP tests. The household passes both income tests and the asset test and is categorically eligible through BBCE, so it receives $994 − 30% × $1,310 = $601 a month." +us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"It used a four-person maximum allotment of $879 instead of the FY2026 value of $994. It also used a slightly low standard deduction, giving net income of $1,316 instead of $1,310. The result was $484 a month instead of $601." +us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"It described the right formula but reached $609.74 a month. That implies a maximum allotment above $994 and no whole-dollar rounding; the correct calculation is $994 − 30% × $1,310 = $601." +us,scenario_038,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $595 a month matches using the FY2025 $204 standard deduction (net $1,329, contribution $399) instead of the FY2026 $223 deduction (net $1,310, contribution $393). The correct benefit is $601." +us,scenario_038,snap,grok-4.3,llm_error,other,False,"It misread the no-inference instruction as a reason not to apply SNAP's program rules at all and entered 0. Program parameters are not household inputs. Applying them gives $994 − 30% × $1,310 = $601 a month." +us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a projected $1,000 maximum allotment instead of $994 and a $225 standard deduction instead of $223, giving $608 a month. It also annualized that inconsistently as $7,280." +us,scenario_038,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction and $1,310 net income, but used a $993 maximum allotment instead of the FY2026 four-person value of $994. That left it $1 a month short: $600 instead of $601." +us,scenario_038,snap,grok-4.7,llm_error,thresholds_rates,False,"It used FY2025 values: an $11,700 annual maximum ($975 a month) and a $2,604 annual standard deduction ($217 a month). The FY2026 values are $994 and $223, so it got $6,963 instead of $7,212." +us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,985 annual standard deduction (about $332 a month) instead of $223 a month, and a $12,337 annual maximum allotment (about $1,028 a month) instead of $994 a month. Both overstated the benefit." +us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"It used about a $1,000 maximum allotment instead of $994 and a standard deduction near $217 instead of $223 (net income $1,316 instead of $1,310). That gave $605 a month instead of $601." +us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $208 standard deduction instead of the FY2026 $223, and a $1,020 maximum allotment instead of $994. That gave $622 a month instead of $601." +us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction but used a $997 maximum allotment instead of $994. It also did not round net income to $1,310 or the benefit to whole dollars, giving $604.04 instead of $601 a month." +us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment instead of the FY2026 four-person value of $994. It also used a smaller standard deduction (net $1,329 instead of $1,310), overstating the benefit at about $651 a month." +us,scenario_038,snap,muse-wonder,llm_error,thresholds_rates,False,"It used a standard deduction of about $217 instead of the FY2026 four-person value of $223. That gave net income of $1,316 instead of $1,310 and a benefit of $599 instead of $601 a month." +us,scenario_038,snap,ox-alpha,llm_error,thresholds_rates,False,"It used a $207 standard deduction instead of the FY2026 $223, so net income came out at $1,326 instead of $1,310 and the benefit at $596 instead of $601 a month. It also annualized imprecisely, giving $7,155." +us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Its own calculation with the 2024 $973 maximum gave $572.54 a month. It then raised that to $879 a month by citing a shelter deduction, even though no rent, mortgage payment or utility costs were given. It also used a $198 standard deduction instead of $223." +us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It set net income to $0 through a shelter deduction, although the household has no listed shelter costs; net income is actually $1,310 after the earned income and standard deductions. It also used a $964 maximum allotment instead of $994." us,scenario_038,spouse_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only the CHIP age and income conditions and skipped the prerequisite that the person not qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes the spouse ineligible for CHIP." us,scenario_038,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated the absence of listed Medicaid coverage as the absence of Medicaid eligibility, despite the question asking about eligibility rather than enrollment. The spouse qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP even though the spouse is under 19 and below the CHIP income limit." us,scenario_038,spouse_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model concluded that satisfying the CHIP age and income ceiling was sufficient and never evaluated Medicaid eligibility first. The spouse is Medicaid-eligible under the OLDER_CHILD category, so the mutually exclusive CHIP pathway yields No." @@ -3525,43 +3578,44 @@ us,scenario_038,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,s us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It invented a $4,500 per-person exemption for all four members, totaling $18,000, and a 1.75% 'post-2024' bottom rate. The post-2024 reform actually set a flat 3% rate with a $25,000 MFJ standard deduction. That deduction exceeds the $21,896 AGI, so tax is $0 rather than $68." us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits. The correct derivation is AGI of about $21,896 minus Louisiana's $25,000 MFJ standard deduction, which gives zero taxable income and $0 tax." us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used the pre-reform $4,500 per-spouse exemptions and $1,000 dependent exemptions, then taxed $10,896 at the pre-2022 2%/4% bracket rates with a wrong $10,000 bracket break. It never applied the 2025 reform's $25,000 MFJ standard deduction and flat 3% rate, which bring taxable income to zero on AGI of $21,896." -us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly applied 5% to a federal EITC on the plateau. However, it used ~$7,478 as the 2026 two-child maximum EITC instead of $7,316, which overstated the Louisiana credit as $374 instead of $365.80." -us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed Louisiana has no refundable EITC and only nonrefundable credits. In fact, Louisiana provides a refundable EITC equal to 5% of the federal EITC. With a $7,316 federal EITC, that yields $365.80." -us,scenario_038,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model asserted that no refundable Louisiana credits apply. It missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80." -us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used the correct 5% rate but put the federal EITC at about $3,680 for MFJ with two children and ~$21,400 of earned income. That income is on the plateau, below the ~$31k MFJ phase-out threshold, so the federal credit is the full $7,316. The Louisiana credit is therefore $365.80." -us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model put the federal EITC at ~$2,640 as if ~$21,900 of earned income were deep in the phase-out range. For a joint filer that income sits on the plateau below the ~$31k MFJ phase-out start, so the federal EITC is $7,316 and 5% of it is $365.80." -us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but rounded the federal EITC down to about $7,000. The 2026 two-child maximum, which this plateau-income household receives, is $7,316, so the credit is $365.80, not $350." -us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model claimed Louisiana eliminated its refundable EITC. Louisiana's refundable EITC remains at 5% of the federal EITC, which gives $365.80 on the $7,316 federal credit." -us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable and returned $0. The Louisiana EITC is refundable at 5% of the federal EITC, paying $365.80 regardless of state tax liability." -us,scenario_038,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model correctly identified the $7,316 federal EITC but applied a 50% Louisiana match instead of the actual 5%. That inflated the credit tenfold, to $3,658 instead of $365.80." -us,scenario_038,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model made two errors: it used a 3.5% Louisiana rate instead of 5%, and a $7,430 federal EITC instead of the 2026 two-child maximum of $7,316. Correctly computed, 5% × $7,316 = $365.80." -us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $7,354 as the federal EITC instead of the 2026 two-child maximum of $7,316. That overstated the credit by $1.90." -us,scenario_038,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said no refundable Louisiana credit applies. It missed the refundable Louisiana EITC, which is 5% of the federal EITC: $365.80 on a $7,316 federal credit." -us,scenario_038,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,The model claimed Louisiana eliminated its state EITC. Louisiana's refundable EITC at 5% of the federal credit is in effect for 2026 and yields $365.80 here. -us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model returned $0 without recognizing the Louisiana refundable EITC. That credit is 5% of the household's $7,316 federal EITC, or $365.80." -us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $7,250 as the federal EITC instead of the 2026 two-child maximum of $7,316. That understated the credit as $362.50 instead of $365.80." -us,scenario_038,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model returned $0 and ignored Louisiana's refundable EITC. That credit equals 5% of the $7,316 federal EITC, or $365.80." -us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly identified the Louisiana EITC as a refundable share of the federal credit. However, $353.43 at 5% implies a federal EITC of about $7,069 instead of the $7,316 two-child plateau maximum for 2026." -us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model recognized the refundable Louisiana EITC, but its $350 implies a $7,000 federal EITC. The household receives the 2026 two-child maximum of $7,316, so 5% gives $365.80." -us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable and zeroed it out because state liability is $0. The LA EITC is refundable at 5% of the federal EITC, which pays $365.80 even with no state tax." -us,scenario_038,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model classified the Louisiana EITC as nonrefundable. The EITC is refundable regardless of state tax liability or child care expenses, and it yields 5% × $7,316 = $365.80." -us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model looked only for listed credit inputs and missed that Louisiana's refundable EITC is computed automatically from the federal EITC. At 5% × $7,316, the credit is $365.80." -us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed no Louisiana refundable credit at this income level. Low-to-moderate earned income is exactly what produces a federal EITC of $7,316, and Louisiana's refundable 5% match is $365.80." -us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model required explicit refundable-credit inputs and returned $0. Louisiana's refundable EITC is derived from the federal EITC the household qualifies for: 5% × $7,316 = $365.80." -us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $8,231, which is the 2026 maximum for three or more children. This household has two qualifying children, so the maximum is $7,316 and the credit is $365.80." -us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the correct $7,316 federal EITC but applied a 3.5% Louisiana rate instead of 5%. That gave $256.06 instead of $365.80." -us,scenario_038,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but put the federal EITC at $7,427 instead of the 2026 two-child maximum of $7,316. That overstated the credit by $5.55." -us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits are triggered. Louisiana's refundable EITC is triggered automatically by the federal EITC and pays 5% × $7,316 = $365.80." -us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model asserted that Louisiana has no state EITC. Louisiana has a refundable EITC at 5% of the federal credit, which is $365.80 for this household." -us,scenario_038,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Louisiana has no refundable EITC. Louisiana's EITC is refundable at 5% of the $7,316 federal EITC, giving $365.80." -us,scenario_038,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used $7,332 as the federal EITC instead of the 2026 two-child maximum of $7,316. It also rounded the result to whole dollars, giving $367 instead of $365.80." -us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model concluded there was no applicable EITC match. Louisiana's refundable EITC matches 5% of the federal EITC and yields $365.80 on this household's $7,316 federal credit." -us,scenario_038,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used ~$7,422 as the federal EITC instead of the 2026 two-child maximum of $7,316. That overstated the credit as $371 instead of $365.80." -us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there is no answer to score. The correct derivation is 5% of the $7,316 federal EITC, which is $365.80." -us,scenario_038,state_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model applied the correct 5% rate to a federal EITC of $6,500. The household's federal EITC is the 2026 two-child plateau maximum of $7,316, so the credit is $365.80, not $325." -us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model tied refundable credits to Louisiana taxable income and school-readiness activities. It missed that the Louisiana EITC is refundable regardless of state taxable income and pays 5% × $7,316 = $365.80." -us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly placed the household on the federal EITC plateau but used $7,430 as the 2026 two-child maximum instead of $7,316. That gave $371.50 instead of $365.80." -us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model stated that Louisiana provides no applicable refundable credits. It missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used the correct 5% rate and correctly found the household on the EITC plateau. But it put the 2026 two-child federal EITC maximum at about $7,478 instead of $7,316, which inflated the result to $374 instead of $365.80." +us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Louisiana has no refundable earned income credit. In fact Louisiana's EITC is refundable and equals 5% of the federal EITC. On the $7,316 federal credit that comes to $365.80, not $0." +us,scenario_038,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model said no refundable Louisiana credit applies and missed the refundable Louisiana EITC, which is 5% of the federal EITC. On this household's $7,316 federal credit that is $365.80." +us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly identified the refundable 5% Louisiana EITC and roughly $21,400 of earned income. It then put the federal EITC at about $3,680, roughly half the real amount. Earned income near $21,900 sits on the two-child plateau, below the joint-filer phase-out threshold, so the federal credit is the full $7,316 and the Louisiana credit is $365.80." +us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model computed combined earned income of about $21,896 correctly, then cut the federal EITC to about $2,640 as though the credit were deep in its phase-out. Joint-filer AGI of about $21,900 is below the 2026 phase-out start, so the federal credit is the full $7,316 and 5% of it is $365.80." +us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the correct 5% refundable rate to a federal EITC of about $7,000. The 2026 maximum for two children, which this plateau household receives, is $7,316, so the answer should be $365.80, not $350." +us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model claimed Louisiana eliminated its refundable EITC. Louisiana still has a refundable EITC of 5% of the federal credit, which gives $365.80 on the $7,316 federal EITC." +us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable. It is refundable at 5% of the federal EITC, so this household gets $365.80 regardless of its Louisiana tax liability." +us,scenario_038,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model had the correct federal EITC of $7,316 but applied a 50% Louisiana match instead of the actual 5%. That inflated the credit tenfold to $3,658 instead of $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used Louisiana's old 3.5% EITC match instead of the current 5%. It also overstated the federal EITC as $7,430 instead of $7,316, producing $260.05 instead of $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the correct 5% refundable rate but put the 2026 two-child federal EITC maximum at $7,354 instead of $7,316. That gave $367.70 instead of $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said no refundable Louisiana credit applies and missed the refundable Louisiana EITC of 5% of the federal credit. On the $7,316 federal EITC that is $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model claimed Louisiana eliminated its state EITC. Louisiana's refundable EITC remains in force at 5% of the federal credit, which is $365.80 for this household." +us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no Louisiana refundable credit applies and missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied the correct 5% rate to a federal EITC of $7,250. The household is on the two-child plateau, where the 2026 maximum is $7,316, so the answer should be $365.80, not $362.50." +us,scenario_038,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model returned $0 and missed the refundable Louisiana EITC of 5% of the federal credit. On this household's $7,316 federal EITC that is $365.80." +us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $353.43 implies a federal EITC of about $7,069 at the 5% rate. The household gets the full 2026 two-child maximum of $7,316, so the answer should be $365.80." +us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $350 implies a federal EITC of about $7,000 at the 5% Louisiana rate. The household's plateau federal credit is the full $7,316, so the answer should be $365.80." +us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated the Louisiana EITC as nonrefundable and capped it at zero liability. It is refundable at 5% of the federal EITC, so it pays $365.80 even with no state tax owed." +us,scenario_038,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model called the Louisiana EITC nonrefundable and zeroed it out because the household owes no state tax. The credit is refundable at 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model looked for explicit refundable-credit inputs and missed that the Louisiana EITC follows automatically from federal EITC eligibility. At 5% of the $7,316 federal credit it is $365.80." +us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed no Louisiana refundable credit at this income. In fact low earned income is exactly what produces the maximum federal EITC of $7,316 here, and Louisiana refunds 5% of it, or $365.80." +us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model required explicit refundable-credit inputs and missed that the Louisiana EITC derives from the federal EITC. It is 5% of the $7,316 federal credit, or $365.80." +us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied the correct 5% rate to a federal EITC of $8,231, which matches the three-or-more-children maximum. This household has two qualifying children and a $7,316 maximum, so the answer should be $365.80." +us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model had the correct federal EITC of $7,316 but applied Louisiana's old 3.5% match instead of the current 5%. That gave $256.06 instead of $365.80." +us,scenario_038,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model used the correct 5% rate but put the federal EITC at $7,427 instead of the 2026 two-child maximum of $7,316. That gave $371.35 instead of $365.80." +us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said no Louisiana refundable credit is triggered and missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model claimed Louisiana has no state EITC. Louisiana has a refundable EITC of 5% of the federal credit, which is $365.80 here." +us,scenario_038,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Louisiana has no refundable EITC. The Louisiana EITC is refundable at 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used the correct 5% rate but a federal EITC of $7,332 instead of $7,316, and it rounded to whole dollars. That gave $367 instead of $365.80." +us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said there was no EITC match. Louisiana does match the federal EITC, refundably, at 5%, giving $365.80 on the $7,316 federal credit." +us,scenario_038,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used the correct 5% refundable rate but put the federal EITC at about $7,422 instead of the 2026 two-child maximum of $7,316. That gave $371 instead of $365.80." +us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for state_refundable_credits. The correct derivation is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model applied the correct 5% rate to a federal EITC of $6,500, well below the 2026 two-child maximum of $7,316 that this plateau household receives. That gave $325 instead of $365.80." +us,scenario_038,state_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model applied the correct 5% rate to a federal EITC of about $7,148 instead of the 2026 two-child maximum of $7,316. That gave $357.40 instead of $365.80." +us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model tied Louisiana refundable credits to taxable income and child care activity, and so missed the refundable Louisiana EITC. That credit is 5% of the federal EITC and pays $365.80 regardless of state tax liability." +us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly placed the household on the plateau and used the 5% rate. But it put the 2026 two-child federal EITC maximum at $7,430 instead of $7,316, which gave $371.50 instead of $365.80." +us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said Louisiana offers no refundable credit for this household and missed the refundable Louisiana EITC, which is 5% of the $7,316 federal EITC, or $365.80." us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model actually reached the correct single-filer result of $8,596, then dropped it. It switched to qualifying-surviving-spouse treatment even though there is no dependent child, used the MFJ $32,000/$44,000 Social Security thresholds (taxable SS $25,187) and the $32,200 joint deduction, and ended at about $4,488." us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It applied qualifying-surviving-spouse joint brackets and the $32,200 joint standard deduction, but that status requires a dependent child and none is listed. The correct treatment is single: $16,100 deduction, taxable income $63,109, with the top slice taxed at 22% above $50,400." us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It overstated AGI at $84,395 instead of $79,209 and invented an extra standard deduction for disability. It then claimed an elderly/disabled credit wipes out the tax, but that credit is fully phased out at this AGI and Social Security level. Correct single taxable income is $63,109, which owes $8,596." @@ -3632,50 +3686,51 @@ us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,age_disability,False us,scenario_039,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage earnings and applied the 7.65% employee Social Security and Medicare rate to an unstated payroll-tax base. The household has no listed wage or salary income, while its listed estate, retirement, IRA, pension, and negative self-employment income are not subject to employee payroll tax, yielding $0." us,scenario_039,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." us,scenario_039,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the $8,750 standard deduction but left out the $930 personal exemption, so its taxable income was $39,770 and its tax about $2,029, not $1,975.80. It then reported $2,192, which none of its own steps produce." -us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"The model gave the surviving spouse the $17,500 married-filing-jointly standard deduction. With no dependent, the head files as single and gets $8,750. That cut taxable income from $38,839.97 to $30,090." -us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model invented a Virginia retirement income credit and disability exclusion that wiped out the tax. Virginia has neither for a 61-year-old receiving IRA, pension and estate income. After the Social Security subtraction, the $8,750 standard deduction and the $930 exemption, $38,839.97 is still taxable." -us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,age_disability,False,"The model applied an age 65+ / disability deduction to zero out the tax. Virginia's age deduction requires age 65, and the disability subtraction covers only disability income, which this household doesn't have. Taxable income stays at $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the surviving spouse as a joint filer and used a $17,000 standard deduction. With no dependent, the head files as single with an $8,750 deduction. Its bracket arithmetic was also inconsistent, giving $1,453 instead of tax on $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model used the outdated $8,500 standard deduction and added an $800 exemption for disability. Virginia's additional $800 exemption is only for age 65+ or blindness. It then cut its own $1,944 result to $1,545 for unspecified 'medical/other adjustments' that don't exist under the standard deduction." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the pre-2025 $8,500 single standard deduction instead of the 2026 amount of $8,750. That overstated taxable income by $250 and the tax by about $14." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model invented a $20,000 disability subtraction for pension and IRA income; Virginia's disability subtraction covers only disability income. It also used the $17,000 joint standard deduction for a single filer and claimed three $800 exemptions instead of one $930 exemption, which drove the tax down to $326." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model left out the $930 personal exemption and used $8,500 instead of the $8,750 standard deduction, which gives $2,044. It then raised that figure to $3,400 for an 'underestimated' Social Security amount, even though Virginia subtracts taxable Social Security in full." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"The model assumed a $17,500 joint standard deduction for the surviving spouse. With no dependent, the head files as single with $8,750, so taxable income is $38,839.97, not $30,090." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $36,105 of Social Security, but federal AGI included only $30,689 of it. It also invented a $12,000 retirement subtraction for disability and used outdated $4,000 and $970 amounts instead of the $8,750 standard deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model never applied Virginia's subtraction of the $30,689.25 of federally taxable Social Security. It taxed $69,529 instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. That produced $39,090 of taxable income instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied an $8,500 standard deduction instead of Virginia's 2026 single amount of $8,750. That overstated taxable income by $250 and the tax by about $14." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an $8,500 standard deduction instead of the 2026 single amount of $8,750. That gives $39,090 of taxable income instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no specific computation. The correct taxable income is $38,839.97, after subtracting taxable Social Security, the $8,750 standard deduction and the $930 exemption. Its $2,420 implies taxable income of about $46,565, so it didn't apply the full deductions." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $1,990 matches the outdated $8,500 single standard deduction. The 2026 amount is $8,750." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $9,000 standard deduction and an $800 personal exemption instead of the 2026 amounts of $8,750 and $930. That gave taxable income of $38,720 instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no reasoning. Its $3,371 implies taxable income of about $63,100, far above the correct $38,839.97, so it didn't fully subtract the $30,689.25 of federally taxable Social Security from federal AGI." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction, which is outdated; the 2026 single amount is $8,750. That overstated taxable income by $750." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $1,990 matches the outdated $8,500 standard deduction. Virginia's 2026 single standard deduction is $8,750." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction instead of the 2026 single amount of $8,750, so taxable income came out at $39,590 instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model handled the Social Security subtraction correctly but used the outdated $8,500 standard deduction instead of $8,750. That gave $39,090 of taxable income." -us,scenario_039,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model left out the $25,950 of estate income and treated the self-employment loss as zero. It applied a flat 2% rate instead of Virginia's graduated brackets, which top out at 5.75%. It then added $2,000 of fictitious estimated payments to reach $2,516." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The model relied on age-based treatment to make the tax base zero or negative. Virginia's age deduction starts at 65, so this 61-year-old gets none, and $38,839.97 remains taxable after the standard deduction and exemption." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed a low-income scenario. Virginia AGI is $48,519.97 even after subtracting Social Security, so $38,839.97 is taxable after the $8,750 standard deduction and $930 exemption." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. That overstated taxable income by $250." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model gave the head a surviving-spouse/joint-based standard deduction. Without a dependent, the head files as single with $8,750. Its $1,760.18 implies taxable income of about $35,090 instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model used a $17,500 joint standard deduction for the surviving spouse. With no dependent, the head files as single with $8,750, so taxable income is $38,839.97, not $30,090." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model applied a 'surviving-spouse' standard deduction larger than the $8,750 single amount. A surviving spouse without a dependent files as single. Its $1,789 implies about $35,590 of taxable income instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"The model used a $17,500 surviving-spouse/joint standard deduction. The head has no dependent and files as single with $8,750." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model taxed about $69,529 because it never subtracted the $30,689.25 of federally taxable Social Security from federal AGI. The correct taxable income is $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"The model used a $17,500 surviving-spouse/joint standard deduction instead of the $8,750 single amount; the head has no dependent. It also miscalculated the bracket tax on its own $30,090, getting $1,442.68 instead of $1,472.68." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction and exemptions wipe out taxable income. Virginia AGI is $48,519.97 after the Social Security subtraction, and the $9,680 of deduction plus exemption leaves $38,839.97 taxable." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an $8,000 single standard deduction instead of the 2026 amount of $8,750, so taxable income was $39,590 instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly ruled out the age deduction but used the outdated $8,500 standard deduction instead of $8,750, which put taxable income at $39,090." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used an $8,500 standard deduction instead of the 2026 single amount of $8,750. That overstated taxable income by $250 and the tax by about $14." -us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction instead of the 2026 single amount of $8,750, which produced $39,590 of taxable income." -us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an $8,000 standard deduction instead of $8,750, so taxable income was $39,590 instead of $38,839.97." -us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model started from an $84,625 AGI without subtracting Social Security. It invented age 65+, disability and surviving-spouse deductions that don't apply to a 61-year-old, then replaced its own ~$4,100 result with an arbitrary $1,500." -us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model wrongly said Virginia's Social Security subtraction requires AGI of $50,000 or less. Virginia subtracts all federally taxable Social Security regardless of income, so $30,689.25 comes out of AGI. It also used the outdated $8,500 standard deduction instead of $8,750." -us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $930 personal exemption as the standard deduction and left out the $8,750 standard deduction. It also built federal AGI with the full $36,105 of Social Security instead of the $30,689 taxable portion. It then reported $3,260.45, which doesn't match its own $1,878.93 computation." -us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"The model invented an 'age-61 exemption' and a nonrefundable 'pension deduction' credit that cancelled the tax. Virginia has neither, so the tax on $38,839.97 of taxable income stands with no nonrefundable credit against it." +us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly arrived at VA AGI of $48,520 and the $8,750 deduction but left out the $930 personal exemption, so it taxed $39,770 and got $2,029. It then reported $2,192, a figure its own arithmetic does not support. The correct taxable income is $38,839.97, which gives $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It gave the head the $17,500 married/joint standard deduction because she is a 'surviving spouse'. With no dependent she files as single in Virginia and gets the $8,750 deduction, so taxable income is $38,839.97, not $30,090." +us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"It invented a Virginia retirement-income credit and a disability-income exclusion that wiped out the tax. Virginia's age deduction requires age 65, and the disability subtraction applies only to disability income, not to IRA, pension or estate income. Neither applies to this 61-year-old, whose taxable income is $38,839.97 after the Social Security subtraction, the $8,750 standard deduction and the $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,age_disability,False,"It assumed an age 65+/disability deduction that eliminates the tax. The head is 61, which rules out the age deduction, and has no disability income, which rules out the disability subtraction. The $48,520 of non-Social Security income is still taxable after the $8,750 standard deduction and the $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It treated the surviving spouse as a joint filer and used a $17,000 standard deduction, but Virginia taxes her as single with an $8,750 deduction. It also got the brackets wrong, putting the tax at $670 on the first $17,000 when the correct figure is $720." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used an outdated $8,500 standard deduction and added an $800 exemption for disability, which Virginia allows only for age 65+ or blindness. That gave $1,944, and it then cut the figure to $1,545 for 'medical/other adjustments' that do not exist because the standard deduction is larger than itemized deductions." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750 and applied the $930 exemption only loosely, ending at about $1,990. The correct taxable income is $38,839.97, which gives $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"It took a $20,000 'disability retirement subtraction' from the IRA and pension income, but Virginia's disability subtraction covers only disability income. It also put the full $36,105 of Social Security into AGI instead of the $30,689 federally taxable amount. On top of that, it used the $17,000 joint deduction and $2,400 of extra exemptions, when the correct amounts are the $8,750 single deduction and one $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It correctly found VA AGI of $48,520 but left out the $930 exemption and used $8,500 instead of $8,750, which gave $2,044. It then raised the figure to $3,400 for an unexplained 'adjustment' that has no basis in Virginia law." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It gave the surviving spouse the $17,500 joint standard deduction, but Virginia taxes her as single with $8,750. That understated taxable income by $8,750 and the tax by about $503." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the full $36,105 of Social Security instead of the $30,689 included in federal AGI and invented a $12,000 retirement subtraction for disability. It also used an obsolete $4,000 standard deduction and a $970 exemption, when the correct amounts are $8,750 and $930." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It never applied Virginia's subtraction of the $30,689.25 of federally taxable Social Security and taxed $69,529 instead of $38,839.97. Its $8,750 deduction and $930 exemption were correct." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. That overstated taxable income by $250 and the tax by $14.38, giving $1,990.18 instead of $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. That gave taxable income of $39,090 instead of $38,839.97 and tax of $1,990.18 instead of $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used an $8,500 standard deduction instead of the 2026 single amount of $8,750. The extra $250 of taxable income at 5.75% accounts exactly for its $14.38 overstatement." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $2,420 implies Virginia taxable income of about $46,600 instead of $38,839.97, which means it left out most of the $8,750 standard deduction and the $930 exemption that apply to VA AGI of $48,519.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,990 matches using the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750. With the $930 exemption, the correct taxable income is $38,839.97, which gives $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $9,000 standard deduction and an $800 personal exemption, but the 2026 single amounts are $8,750 and $930. Its taxable income of $38,720 is understated by about $120." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $3,371 implies Virginia taxable income of about $63,100, which means it did not fully subtract the $30,689.25 of taxable Social Security from federal AGI before applying the $8,750 deduction and the $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used an $8,000 standard deduction instead of the 2026 single amount of $8,750, which gave taxable income of $39,590. It also miscalculated the tax on that base, which is $2,018.93, not the $2,028.93 it reported." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $1,990 matches using the outdated $8,500 single standard deduction instead of $8,750. That overstated taxable income by $250 compared with the correct $38,839.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used an $8,000 standard deduction instead of the 2026 single amount of $8,750, which put taxable income at $39,590 instead of $38,839.97 and overstated the tax by about $43." +us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It got the federal taxable Social Security and VA AGI right but used the outdated $8,500 standard deduction instead of the 2026 amount of $8,750. That overstated the tax by $14.38." +us,scenario_039,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It left out the $25,950 of estate income, set the self-employment loss to zero, and applied a flat 2% rate instead of Virginia's 2%–5.75% graduated brackets. It then added $2,000 of 'prior-year estimated payments' that are not in the facts and have nothing to do with liability." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"It assumed an age-based Virginia benefit made the tax base non-positive, but the age deduction requires age 65 and the head is 61. VA AGI of $48,519.97, less the $8,750 deduction and the $930 exemption, leaves $38,839.97 taxable." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated the household as low-income with no tax, ignoring $48,520 of non-Social Security income that Virginia taxes. After the $8,750 deduction and the $930 exemption, that income produces $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used an $8,500 single standard deduction instead of the 2026 amount of $8,750, which gave taxable income of $39,090 instead of $38,839.97 and tax of about $1,990 instead of $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It applied a surviving-spouse/joint standard deduction, which its $1,760.18 implies was about $12,500, instead of the $8,750 single deduction. Virginia has no surviving-spouse filing status, and she has no dependent, so she files as single." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It used the $17,500 joint standard deduction for the surviving spouse, but Virginia taxes her as single with $8,750. That cut taxable income to $30,090 instead of $38,839.97." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It applied a 'surviving-spouse' standard deduction larger than the single amount (its $1,789 implies about $12,000) instead of the $8,750 single deduction that applies to a widow with no dependent filing in Virginia." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It gave the surviving spouse the $17,500 joint standard deduction. Virginia taxes her as single with an $8,750 deduction, so taxable income is $38,839.97, not $30,090." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $69,529 shows it left out Virginia's subtraction of the $30,689.25 of federally taxable Social Security. Its deduction and exemption were correct, but it applied them to the full federal AGI." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used the $17,500 joint deduction for the surviving spouse instead of the $8,750 single deduction. It also miscalculated the tax on its own $30,090 base, which is $1,472.68, not $1,442.68." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and exemptions cancel out all taxable income. Those total only $9,680, against $48,519.97 of VA AGI, which leaves $38,839.97 taxable." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used an $8,000 single standard deduction instead of the 2026 amount of $8,750, which put taxable income at $39,590 instead of $38,839.97." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly found VA AGI of $48,520 but used the outdated $8,500 standard deduction instead of the 2026 amount of $8,750, overstating the tax by about $14." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied the outdated $8,500 single standard deduction instead of $8,750, which gave taxable income of $39,090 instead of $38,839.97 and tax of $1,990.18 instead of $1,975.80." +us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an $8,000 standard deduction instead of the 2026 single amount of $8,750, overstating taxable income by $750 and the tax by about $43." +us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied an $8,000 standard deduction instead of the 2026 single amount of $8,750, which put taxable income at $39,590 instead of $38,839.97." +us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It put the full $36,105 of Social Security in AGI without subtracting it and invented age-65, disability and surviving-spouse deductions that do not apply to a 61-year-old. It then gave an arbitrary $1,500 that matches neither its own ~$4,100 estimate nor the correct computation." +us,scenario_039,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used the outdated $8,500 single standard deduction instead of the 2026 amount of $8,750, which gave taxable income of $39,090 instead of $38,839.97." +us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It wrongly claimed Virginia's Social Security subtraction requires AGI of $50,000 or less. Virginia subtracts all federally taxable Social Security regardless of income, so it overstated taxable income by $30,689. It also used $8,500 instead of the $8,750 standard deduction." +us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It put the full $36,105 of Social Security in federal AGI instead of the $30,689.25 taxable portion and used $930 as the standard deduction, leaving out the $8,750 deduction entirely. It calculated $1,878.93 and then reported an unrelated $3,260.45." +us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It invented an 'age-61 exemption' of $11,898 and treated the $2,030 of pension income as a nonrefundable credit against tax, and neither exists in Virginia law. The correct path subtracts taxable Social Security, the $8,750 deduction and the $930 exemption, leaving $1,975.80 of tax." us,scenario_039,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed AGI correctly at $32,300 but assumed the TCJA sunset took effect in 2026, subtracting a $18,850 standard deduction plus two revived personal exemptions of $10,600 for $29,450 of offsets. Personal exemptions remain repealed in 2026 and the MFJ standard deduction is $32,200, plus $3,300 of aged additional standard deduction and the $12,000 OBBBA senior deduction for two 65+ spouses, totaling $47,500 — more than AGI, so taxable income is $0 rather than the $2,850 it taxed at 10%." us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model reached the correct $32,300 AGI but then paired a $20,000 MFJ standard deduction with $10,600 of personal exemptions, a combination that exists under no 2026 rule set — personal exemptions are permanently repealed and the 2026 MFJ standard deduction is $32,200. Adding the $3,300 aged additional standard deduction and the $12,000 senior deduction for two spouses aged 65+ wipes out the entire $32,300 of AGI, leaving $0 taxable income instead of the $1,700 it taxed at 10%." @@ -3709,104 +3764,89 @@ us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error, us,scenario_040,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed $21,420 of Social Security as part of the Arizona base, missing both that provisional income of $40,600 puts only $4,300 of the $25,200 in federal AGI and that Arizona subtracts 100% of the Social Security benefits included in federal AGI. It also contradicted its own arithmetic, computing $484.25 and then submitting $486.25; with Social Security removed, Arizona AGI is $28,000, below the roughly $32,200 MFJ standard deduction, so taxable income and tax are both zero." us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Arizona’s distinct excise-tax and property-tax credits and treated age plus an asserted $25,000 AGI ceiling as sufficient for a $200 refundable credit. It never applied the specific eligibility requirements for either Arizona credit; this household qualifies for neither, so az_refundable_credits is $0." us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted state_refundable_credits entirely instead of submitting the required numeric value and explanation. The required output is $0 because every state and locality refundable-credit component, including Arizona’s, evaluates to zero." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left the $928 of qualified dividends out of its non-Social Security income ($32,807 instead of $33,735). That understated provisional income, taxable Social Security ($16,057 instead of $16,846) and AGI ($48,864 instead of $50,581). Taxable income therefore fell to $24,714 instead of $26,431, and ordinary tax came out low at $2,133." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse and used the $32,200 joint standard deduction, the $1,650 married aged add-on and joint brackets. QSS requires a dependent child, so the head files Single with a $16,100 + $2,050 + $6,000 deduction. It also taxed the $3,753 of capital gains as ordinary income instead of putting them in the 0% net-capital-gain bucket." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted the full $29,580 of Social Security in income instead of the $16,846 taxable under the §86 formula, used a made-up $28,900 surviving-spouse deduction, and left out the $6,000 senior deduction. It also taxed $4,681 of qualified dividends and gains at 15% even though they fall in the 0% band, and then subtracted a nonexistent 'dependent exemption credit' of $783." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the deductions wiped out taxable income without doing the computation. AGI is $50,581 against $24,150 of total deductions, which leaves $26,431 of taxable income. $21,750 of that is ordinary income taxed at 10%/12% for $2,362, not zero." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security dependent benefits as belonging to someone else and not taxable to the head, which dropped the $16,846 of taxable Social Security from AGI. It also left out the $928 of qualified dividends and the $6,000 OBBBA senior deduction. These offsetting errors produced $1,359." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used a Qualifying Surviving Spouse standard deduction ($30,700 + $1,650) and joint brackets even though there is no dependent child, so the head files Single. It also set taxable Social Security at the 85% cap ($25,143) instead of the formula amount of $16,846, left out the $6,000 senior deduction, and then cut its own $2,603 figure to $1,245 with no computation." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It took 85% of Social Security ($25,143) as taxable instead of applying the lesser-of formula: 0.85×($48,525−$34,000)+$4,500 = $16,846. That inflated AGI to about $58,878. Its final $2,699 also does not follow from the $34,700 of taxable income it stated." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse and applied the $32,200 joint standard deduction, the $1,650 aged add-on and joint brackets. With no dependent child, the head files Single, so the base deduction is $16,100 + $2,050 and $21,750 of ordinary income is taxed at 10%/12%." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction for taxpayers 65 and older and used a guessed $17,750 standard deduction instead of $18,150. It also added the $71 of BDC income to gross income. Taxable income came out around $32,949 instead of $26,431." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as not taxable to the head, which dropped $16,846 of taxable Social Security. It then reported $0 even though its own figure of about $16,792 of taxable income would produce several hundred dollars of 10%/12% tax." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with the $32,200 joint standard deduction, the $1,650 add-on and joint brackets. Without a dependent child the head files Single with $24,150 of total deductions, which leaves $21,750 of ordinary income at 10%/12% instead of $6,050 at 10%." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction and used 2025-level amounts ($15,650 + $2,000) instead of the 2026 $16,100 + $2,050. That overstated taxable income by about $6,500, to $33,063 instead of $26,431." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It deducted only a $17,425 standard deduction, leaving out the $6,000 senior deduction and understating the 2026 base plus aged amount ($18,150). Ordinary taxable income came out at $28,536 instead of $21,750." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 sunset law, with an $8,600 standard deduction, a $5,500 personal exemption and a 15% bracket. OBBBA made the TCJA schedule permanent, so the correct deductions are $16,100 + $2,050 + the $6,000 senior deduction with 10%/12% brackets and no exemption." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used a $17,350 standard deduction and left out the $6,000 OBBBA senior deduction. It also understated the 2026 single-plus-aged amount ($18,150), so ordinary taxable income was $28,682 instead of $21,750." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired, using a $10,300 deduction, a $5,200 personal exemption and a 15% bracket. Under OBBBA's permanent extension, the 2026 single filer gets $16,100 + $2,050 + $6,000 in deductions, with 10%/12% brackets." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used head-of-household status even though the head has no qualifying person. Its $3,156 is consistent with about $33,000 of taxable income, meaning it left out the $6,000 senior deduction that brings taxable income down to $26,431." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It got AGI right ($50,581) but subtracted only the elderly single standard deduction, leaving out the $6,000 OBBBA senior deduction. The $3,500 result implies roughly $31,000 taxed at ordinary rates instead of $21,750." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 brackets and personal exemptions ($15,150 of combined deductions). OBBBA made the TCJA parameters permanent, so the correct deductions are $24,150 and the brackets are 10%/12%." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path taxes $21,750 of ordinary income at 10%/12% ($2,362) and $4,681 of qualified dividends and gains at 0%. The $2,878 answer is consistent with taxing roughly $4,300 more as ordinary income, such as the qualified dividends and gains taxed at 12% instead of the 0% rate." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse, using the $32k/$44k joint Social Security thresholds (taxable Social Security $9,846) and a $31,550 joint-style standard deduction. With no dependent child the head files Single, so taxable Social Security is $16,846 and total deductions are $24,150." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the elderly single standard deduction from $50,581 of AGI and left out the $6,000 OBBBA senior deduction. Its explanation also states $3,584, which does not match the submitted $3,090." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and left out the $6,000 senior deduction. The $3,734 implies about $33,000 taxed at 10%/12%, against the correct $21,750 of ordinary taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction (deducting only $17,850). It also mis-added non-Social Security income as $34,806 instead of $33,806, which inflated taxable Social Security to $17,757 and AGI to $52,563." -us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It gave no computation and called the Social Security 'largely nontaxable'. In fact $16,846 of the $29,580 is taxable, which leaves $26,431 of taxable income and $2,362 of tax. Its $1,234.56 is a placeholder-style figure." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that the standard and age-related deductions fully offset income. AGI of $50,581 less $24,150 of deductions leaves $26,431 of taxable income, including $21,750 of ordinary income taxed at 10%/12%." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits brought tax to zero. No nonrefundable credit applies, and $26,431 of taxable income remains after the $24,150 of deductions, producing $2,362 of tax." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $33,200 surviving-spouse standard deduction with joint brackets. With no dependent child the head files Single, so deductions total $16,100 + $2,050 + $6,000 = $24,150. It also left out the senior deduction it would otherwise have claimed." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It made 85% of Social Security taxable, pushing AGI to $58,949 instead of $50,581. It also taxed the qualified dividends and capital gains at 15%, even though total taxable income of $26,431 keeps all $4,681 in the 0% band." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the qualifying-surviving-spouse standard deduction and joint brackets. With no dependent child, the head files Single with an $18,150 base-plus-aged deduction, which leaves $21,750 of ordinary income instead of $6,036." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the surviving-spouse (joint) standard deduction, which put all ordinary income in the 10% bracket. As a Single filer, the head has $21,750 of ordinary income taxed at 10% up to $12,400 and 12% above." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse with a $33,850 age-adjusted joint standard deduction. With no dependent child, the head files Single with $16,100 + $2,050 + $6,000, leaving $21,750 of ordinary taxable income instead of $6,036." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $34,785 of taxable income implies about $25,000 of taxable Social Security, the 85% cap, instead of the formula amount of $16,846. It also put only the $928 of qualified dividends at 0% and taxed the $3,753 of capital gains as ordinary income." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It added the $71 of qualified BDC income as separate gross income on top of the dividends. That raised provisional income and taxable Social Security by $60.35, so AGI went up by about $131 and tax by about $16 ($2,377.79 instead of $2,361.96)." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used surviving-spouse (joint) deductions and concluded that only 0%-rate preferential income remained. As a Single filer with $24,150 of deductions, the head still has $21,750 of ordinary taxable income." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction (plus QBI and mortgage interest) and left out the $6,000 OBBBA senior deduction for those 65 and older. The $3,125 implies about $28,100 of ordinary taxable income instead of $21,750." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rules: a $10,507 standard deduction, a $5,387 personal exemption and 10%/15% brackets. OBBBA kept the TCJA schedule, so the correct figures are a $16,100 + $2,050 standard deduction, the $6,000 senior deduction and 10%/12% brackets." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration parameters: an $8,300 standard deduction, a $5,300 exemption and a 15% bracket. 2026 law under OBBBA provides $16,100 + $2,050 + a $6,000 senior deduction with 10%/12% brackets." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied the pre-TCJA 2026 sunset schedule ($8,300 standard deduction, $2,000 aged amount, $5,300 personal exemption, 15% bracket). OBBBA made the TCJA rates and the $16,100 standard deduction permanent and added the $6,000 senior deduction." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used reverted pre-TCJA parameters (a $10,233 deduction and a 15% bracket). It also overstated taxable Social Security at $24,205 instead of $16,846 and taxed the $3,753 of capital gains as ordinary income, which brought taxable income to $47,778 instead of $26,431." -us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction and deducted only $18,150. It also counted the farm loss twice (−$2,254), even though 'farm income' and 'farm operations income' describe the same −$1,127 item, which understated taxable Social Security and AGI." -us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer or explanation for this variable, so there was no value to score." -us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed provisional income correctly at $48,525 but then reported $24,108.75 of taxable Social Security. The single-filer §86 formula gives 0.85×($48,525−$34,000)+$4,500 = $16,846.25, so AGI was overstated by $7,262.50." -us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the joint $32k/$44k Social Security thresholds and a $32,150 surviving-spouse standard deduction. With no dependent child the head files Single, so taxable Social Security is $16,846 and deductions are $18,150 plus the $6,000 senior deduction." -us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with MFJ rates, joint Social Security thresholds and a $32,200 standard deduction. Single status applies because there is no dependent child. It also treated the $3,753 of gains as short-term ordinary income instead of 0%-rate net capital gain." -us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse with no dependent child and used a $31,675 deduction and joint brackets, leaving out the $6,000 senior deduction. It also added the $4,700 of tax-exempt IRA distributions to provisional income and reported $25,698 of taxable Social Security, which exceeds the 85% cap of $25,143." -us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $21,824 is about 43% of the $50,581 AGI. The correct computation taxes $21,750 of ordinary income at 10%/12% for $2,362." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model left the $928 of qualified dividends out of gross income, putting non-SS income at $32,807 instead of $33,735. That made taxable Social Security $16,057 instead of $16,846 and taxable income $24,714 instead of $26,431. It also fudged the bracket arithmetic to land on $2,133." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model used qualifying-surviving-spouse joint parameters even though the head has no dependent child: a $32,200 standard deduction plus a $1,650 aged addition. The correct figure is the single filer's $16,100 plus $2,050. It also taxed the $3,753 of non-Schedule-D gains as ordinary income. Together these cut taxable income to about $10,717 instead of $26,431." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model counted essentially all $29,580 of Social Security as income instead of the $16,846 taxable portion and used a made-up $28,900 surviving-spouse deduction instead of $24,150. It taxed the $4,681 of preferential income at 15% when that income falls in the 0% bracket. It also subtracted a nonexistent $783 'dependent exemption credit'." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed taxable income was near zero. In fact, AGI of $50,581, including $16,846 of taxable Social Security, exceeds the $24,150 of standard and senior deductions by $26,431. The $21,750 ordinary portion of that produces $2,361.96 of tax." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the $29,580 of Social Security as belonging to a dependent and excluded the $16,846 taxable portion from the head's AGI. It also dropped the $928 of qualified dividends from income and left out the $6,000 senior deduction, which produced a taxable income of $14,657." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model made the full 85% of benefits ($25,143) taxable instead of applying the formula 0.85×(provisional income − $34,000) + $4,500, which gives $16,846. It used a qualifying-surviving-spouse deduction of $32,350 with no qualifying child, left out the $6,000 senior deduction, and then lowered its result to $1,245 without justification." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model assumed 85% of benefits ($25,143) were taxable instead of computing $16,846 with the single $25k/$34k tiered formula. That inflated AGI to about $58,878 instead of $50,581. Its final $2,699 also does not follow from its own taxable income of about $34,700." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model used qualifying-surviving-spouse status, with a $32,200 joint standard deduction and joint brackets, even though no dependent child exists. The head files single with an $18,150 aged standard deduction, so the model understated taxable income at $10,717 instead of $26,431." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly chose single status and the tiered Social Security computation, but it deducted only an estimated $17,750 standard deduction. It left out the $6,000 OBBBA senior deduction for filers 65 and older, which leaves about $6,000 too much in the 12% bracket. It also counted the $71 of BDC income as extra income." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $16,846 of taxable Social Security on the theory that dependent benefits belong to someone else. It then concluded $0 tax even though its own taxable income of $16,792 had more than $12,000 of ordinary income in the 10%/12% brackets." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the head as a qualifying surviving spouse and applied the $32,200 joint standard deduction plus $1,650. With no dependent child, the head files single, with $16,100 plus $2,050, which puts taxable income at $26,431, not $10,731." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model deducted only a $17,650 standard deduction and left out the $6,000 OBBBA senior deduction. It also added the $71 of BDC income on top of dividends, overstating taxable income at $33,063 instead of $26,431." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $17,425 standard deduction and left out the $6,000 senior deduction for taxpayers 65 and older. That left about $6,700 of extra ordinary income in the 12% bracket." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA provisions had sunset in 2026. It applied pre-TCJA standard deductions, a $5,500 personal exemption, and the 15% bracket, but OBBBA made the TCJA rate schedule permanent and added the $6,000 senior deduction. Under current law, deductions total $24,150 and ordinary rates are 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model applied only a $17,350 standard deduction and left out the $6,000 senior deduction, which overstated taxable income at $33,363 instead of $26,431." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied pre-TCJA sunset law, with a $10,300 standard deduction, a $5,200 personal exemption, and the 15% bracket. OBBBA kept the TCJA schedule in force, so the correct deductions are $18,150 plus the $6,000 senior deduction, with 10%/12% rates." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a head-of-household elderly standard deduction even though there is no qualifying person, and it left out the $6,000 senior deduction. Its $3,156 matches tax on taxable income of about $32,400, not $26,431." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from the correct AGI of about $50,581 but subtracted only the elderly single standard deduction and left out the $6,000 OBBBA senior deduction. Its $3,500 is also above the roughly $3,082 that the $18,150 deduction alone would produce." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied pre-TCJA 2026 brackets and personal exemptions, with combined deductions of $15,150 and a 15% bracket. Current law applies the 10%/12% schedule and a $24,150 total deduction including the senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct path is AGI of $50,581 minus $24,150 of standard and senior deductions, leaving $21,750 of ordinary income at 10%/12% for $2,361.96. The $2,878 answer reflects about $4,300 too little in deductions, consistent with only partly applying the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the head as a qualifying surviving spouse. It applied the joint $32k/$44k Social Security thresholds, which gave $9,846 taxable instead of $16,846, and a $31,550 joint standard deduction. With no dependent child, the head files single." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model had AGI right at about $50,581 but subtracted only the single 65+ standard deduction and left out the $6,000 senior deduction. Its explanation states $3,584 while it submitted $3,090, and both exceed the correct $2,361.96." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model's $3,734 means it taxed roughly $9,000 more ordinary income than the correct $21,750. That is consistent with subtracting only the standard deduction and leaving out the $6,000 senior deduction for filers 65 and older." +us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model mis-added non-SS income as $34,806, which also double-counted the BDC income, and that inflated taxable Social Security to $17,757. It used a $17,850 standard deduction with no $6,000 senior deduction, which produced a taxable income of $34,713." +us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model gave a placeholder figure of $1,234.56 with no computation. The correct path gives taxable income of $26,431 after $24,150 of deductions, $21,750 of it ordinary income, for $2,361.96 of tax." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed the standard and age deductions fully offset income. In fact, $16,846 of Social Security is taxable and AGI of $50,581 exceeds the $24,150 of deductions by $26,431, which leaves $2,361.96 of tax." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and credits erase all tax. No nonrefundable credit applies, and $26,431 of taxable income remains after the $24,150 of standard and senior deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied a $33,200 qualifying-surviving-spouse standard deduction even though there is no dependent child, and it left out the $6,000 senior deduction. The single aged deduction of $18,150 plus $6,000 leaves $21,750 of ordinary income, not $12,700." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model made the full 85% of benefits taxable, giving AGI of $58,949 instead of $50,581, and taxed the qualified dividends and capital gains at 15%. At $26,431 of taxable income, those gains fall entirely in the 0% bracket." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the qualifying-surviving-spouse standard deduction and joint brackets with no qualifying child. That cut ordinary taxable income to about $6,036 instead of the $21,750 that results under single filing status." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model applied the surviving-spouse joint standard deduction, which left taxable income inside the 10% bracket. A surviving spouse without a dependent child files single, which leaves $21,750 of ordinary income taxed at 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model used qualifying-surviving-spouse status, with a $33,850 age-adjusted joint standard deduction, even though no dependent child exists. Single status gives $18,150 plus $6,000 and ordinary taxable income of $21,750, not $6,036." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's taxable income of $34,785 implies AGI of about $58,935, which comes from making the full 85% of benefits ($25,143) taxable instead of $16,846. It also treated only the $928 of qualified dividends as preferential and taxed the $3,753 of capital gains as ordinary income." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model followed the correct structure but added the $71 of qualified BDC income on top of the dividends that already include it, and it rounded the farm loss to $1,127. That raised non-SS income by about $71 and taxable Social Security by about $61. The roughly $132 of extra ordinary income at 12% accounts for the $15.83 overstatement." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used surviving-spouse status with joint Social Security thresholds and the joint standard deduction, so only preferential income was left to tax. Without a dependent child, the head files single, which leaves $21,750 of ordinary income and $2,361.96 of tax." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $3,125 matches subtracting only the roughly $18,150 aged standard deduction from AGI of about $50,600. It left out the $6,000 OBBBA senior deduction, which removes about $720 of 12%-bracket tax." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied pre-TCJA sunset rules: a $10,507 standard deduction, a $5,387 personal exemption, and 10%/15% brackets. OBBBA made the TCJA schedule permanent and added the $6,000 senior deduction, so the correct deductions total $24,150, taxed at 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA 2026 parameters: an $8,300 standard deduction, a $2,050 aged addition, a $5,300 personal exemption, and a 15% bracket. Current law gives an $18,150 aged standard deduction, the $6,000 senior deduction, and 10%/12% rates." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model applied the expired pre-TCJA schedule, with an $8,300 standard deduction, a $5,300 exemption, and a 15% bracket, instead of the permanent TCJA/OBBBA schedule. Under that schedule, deductions total $24,150, ordinary income is $21,750, and rates are 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA deductions and brackets and miscomputed taxable Social Security as $24,205 instead of $16,846. It also taxed the $3,753 of capital gains as ordinary income at 15%, which inflated taxable income to $47,778." +us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model counted the farm loss twice, at −$2,254 instead of −$1,127.29, and subtracted only the $18,150 standard deduction, leaving out the $6,000 senior deduction. That gave a taxable income of $30,478 instead of $26,431." +us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for federal_income_tax_before_refundable_credits. +us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"In the Social Security formula, the model added 50% of the entire excess of provisional income over $25,000 ($11,762.50) instead of capping that second-tier amount at $4,500. That gave $24,108.75 of taxable benefits instead of $16,846.25 and overstated taxable income by about $7,263." +us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied joint ($44k) Social Security thresholds and a $32,150 surviving-spouse standard deduction even though the head has no dependent child. It also left out the $6,000 senior deduction. Under single status, the head has $26,431 of taxable income, not $10,063." +us,scenario_042,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model subtracted only the 65+ standard deduction and left out the $6,000 OBBBA senior deduction. It also added the $71 of BDC income as extra AGI, which leaves about $6,000 too much ordinary income in the 12% bracket." +us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used MFJ rates, joint Social Security thresholds, and the $32,200 joint standard deduction for a surviving spouse with no dependent child, who must file single. It also treated the $3,753 of non-Schedule-D gains as short-term ordinary income." +us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used qualifying-surviving-spouse status with no qualifying child and added the $4,700 of tax-exempt IRA distributions to provisional income. It made $25,698 of benefits taxable, which exceeds the 85% cap of $25,143. It also left out the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model gave $21,824 with no derivation, which exceeds 40% of the household's entire AGI. The correct computation taxes $21,750 of ordinary income at 10%/12% and $4,681 of preferential income at 0%, for $2,361.96." us,scenario_042,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no employee wages were listed and that payroll tax should therefore be zero, but submitted $1,530 anyway. That amount contradicts its own payroll-tax-base reasoning and has no listed wage base supporting it." us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no payroll-tax output. It failed the required structured-output contract rather than completing the computation from the zero listed wage base. us,scenario_042,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"Its own steps applied a $24,000 retirement exclusion that cut Wisconsin income to about $9,700, and it said the standard deduction fully covered that income, which means zero tax. It still submitted $953, a number that follows from neither its own computation nor the reference path (WI AGI $29,981.71, $16,360.34 taxable, $584.74 tax, less the $300 school property tax credit)." -us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as a joint filer for Wisconsin and used the married-joint sliding standard deduction (~$25,728) instead of the single-schedule $12,671.36, which cut taxable income to ~$7,057. It also set nonrefundable credits to zero and missed the $300 school property tax credit (12% of the first $2,500 of the $5,011 in real estate taxes)." -us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It left the $29,580 of Social Security in Wisconsin income and used a ~$9,520 standard deduction, producing ~$55,000 of taxable income, then cancelled the tax with a homestead credit. Wisconsin subtracts all taxable Social Security (WI AGI $29,981.71). The homestead credit is refundable and not available at this income; the only nonrefundable property-tax credit is the $300 school property tax credit, applied against $584.74 of tax." -us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,state_local_rule,False,"It reached $0 by excluding the IRA and pension income under a Wisconsin senior retirement exclusion. The reference subtracts only the $13,655.95 of taxable Social Security and no retirement income, leaving WI AGI of $29,981.71, $16,360.34 taxable, and $284.74 after the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement subtraction that Wisconsin only allows when federal AGI is under $15,000 for a single filer, used a ~$11,790 qualifying-widow standard deduction, and treated the exemptions as $20 credits. Even on its own numbers, $595 minus the $300 school property tax credit and $40 is $255, not $0; the reference nets $584.74 − $300 = $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It computed a ~$731 tentative tax, then claimed without any arithmetic that real-estate and mortgage 'credit interactions' bring it to $0. Property taxes do not count toward Wisconsin's itemized deduction credit and no mortgage interest was reported, so the only nonrefundable offset is the $300 school property tax credit, which the reference subtracts from $584.74 to get $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It used a Wisconsin standard deduction of only ~$5,000, leaving ~$27,500 taxable. The single sliding-scale deduction at the reference's $29,981.71 WI AGI is $12,671.36, which with $950 of exemptions leaves $16,360.34 taxable and $584.74 of tax before the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It subtracted $24,000 of IRA and pension income under Wisconsin's age-67+ retirement income exclusion, plus a 30% capital-gain exclusion, which dropped Wisconsin income to ~$8,600 and the tax to $0. The reference subtracts only the $13,655.95 of taxable Social Security, with no retirement subtraction, so WI AGI is $29,981.71, tax is $584.74, and the result is $284.74 after the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It invented a $10,000 surviving-spouse retirement subtraction, treated the surviving spouse as a joint filer with a ~$21,151 standard deduction, and applied the $700 exemption as a $700 tax credit. In the reference this filer uses the single sliding standard deduction ($12,671.36), and the $700 + $250 exemptions reduce income rather than tax; the $5,000 retirement subtraction is not available above $15,000 of federal AGI. That leaves $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It computed ~$790 of tax, then claimed the itemized deduction credit on real estate taxes and medical costs wipes it out. Property taxes do not count toward Wisconsin's itemized deduction credit and $350 of medical costs is below the floor, so the only nonrefundable credit is the $300 school property tax credit, which leaves a positive tax ($284.74 in the reference)." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It followed the right structure (subtract taxable Social Security, sliding standard deduction, $950 exemptions, 3.5%/4.4% brackets) but stopped at the $790 tentative tax and never subtracted the $300 school property tax credit (12% of the first $2,500 of the $5,011 in real estate taxes). Its WI income of $33,735 also includes the $3,753 of capital gains, compared with the reference's $29,981.71." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It used a ~$14,800 standard deduction with an invented 'elderly addition' in place of the phased single deduction plus the $700 + $250 exemptions. It applied no nonrefundable credit, missing the $300 school property tax credit that the reference subtracts from $584.74." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It used the pre-2023 Wisconsin rates of 3.54%/4.65%, took an unphased $13,410 standard deduction, skipped the $700 + $250 exemptions, and said no nonrefundable credits apply, missing the $300 school property tax credit. The reference taxes $16,360.34 at 3.5%/4.4% for $584.74, then subtracts the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement subtraction that Wisconsin only allows when federal AGI is under $15,000 for a single filer, dropped the $250 age-65 exemption, and never applied the $300 school property tax credit. The reference subtracts only taxable Social Security (WI AGI $29,981.71), uses $950 of exemptions, and nets $584.74 − $300 = $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,837 standard deduction, far below the single sliding-scale amount of $12,671.36, left out the $950 of personal and age exemptions, and applied no school property tax credit. That gave $24,969 taxable and $967 of tax, against the reference's $16,360.34 taxable and $284.74 net." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"Its Wisconsin taxable income of ~$29,446 is essentially Wisconsin AGI with neither the $12,671.36 sliding standard deduction nor the $950 of exemptions taken out, and it then subtracted an invented $50 personal exemption credit. The reference reduces WI AGI of $29,981.71 to $16,360.34 taxable before applying the 3.5%/4.4% rates and the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,137 matches taxing about $28,900 at 3.5%/4.4% with essentially no standard deduction or exemptions and no $300 credit. In the reference, the $12,671.36 sliding deduction and $950 of exemptions reduce WI AGI of $29,981.71 to $16,360.34 taxable, giving $584.74 of tax and $284.74 after the credit." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It subtracted Social Security and a standard deduction but left out the $700 + $250 exemptions and the $300 school property tax credit. $860 matches about $22,600 of taxable income at 3.5%/4.4% with no credit, against the reference's $16,360.34 taxable and $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It applied the $300 school property tax credit correctly, but its $25,235 of taxable income implies a sliding standard deduction of only ~$8,500 and no $950 of personal and age exemptions. At the reference's $29,981.71 WI AGI the single sliding deduction is $12,671.36, which leaves $16,360.34 taxable and $584.74 of tax." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,973 matches Wisconsin taxable income in the $40,000-plus range, which only happens if taxable Social Security stays in the base and the standard deduction, exemptions, and $300 credit are skipped. Wisconsin subtracts the $13,655.95 of taxable Social Security, leaving WI AGI of $29,981.71 and $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It used the right structure (subtract taxable Social Security, standard deduction, senior exemption) on WI AGI of $33,735 but never applied the $300 school property tax credit, so its $786 is a pre-credit tax. The reference applies the $300 credit to $584.74 of tax on $16,360.34 taxable (WI AGI $29,981.71)." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation states a net tax of $1,617 but it submitted $1,348. After adding back a $300 credit, both figures require Wisconsin taxable income above $40,000, which means taxable Social Security stayed in the base. The reference subtracts the $13,655.95 of taxable Social Security and reaches $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,428 after the school property tax credit implies a ~$1,728 tentative tax, which requires Wisconsin taxable income of about $42,000 — only reachable if the $13,655.95 of taxable Social Security is left in the base. The reference subtracts it, giving WI AGI of $29,981.71, $16,360.34 taxable, and $284.74 after the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It subtracted essentially all IRA and pension income as a Wisconsin senior retirement subtraction, which cut WI AGI to ~$9,720 and the tax to $0. The reference subtracts only the $13,655.95 of taxable Social Security, with no retirement subtraction, leaving WI AGI of $29,981.71 and $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It gave a placeholder-like $123.45 with no computation, assuming a retirement subtraction and credits leave only a small amount. The reference subtracts only taxable Social Security, leaving $16,360.34 taxable, $584.74 of tax, and $284.74 after the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It reached $0 by applying pension exclusions on top of the standard deduction. The reference applies no retirement subtraction, only the $13,655.95 taxable Social Security subtraction, leaving $16,360.34 taxable and $284.74 after the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It claimed without any computation that deductions and credits offset all of the tax. The $12,671.36 standard deduction and $950 of exemptions leave $16,360.34 taxable on WI AGI of $29,981.71, and the $300 school property tax credit covers only part of the resulting $584.74." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It treated the school property tax credit on $5,011 of real estate taxes as large enough to wipe out the tentative tax. The credit is 12% of the first $2,500 of property taxes, capped at $300, while tax on $16,360.34 taxable is $584.74, so $284.74 remains." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"It said no Wisconsin nonrefundable credit applies, missing the $300 school property tax credit on $5,011 of real estate taxes. Its ~$800 is a pre-credit tax on WI income that includes the $3,753 of capital gains, above the reference's $584.74 on $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It applied Wisconsin's age-67+ retirement-income subtraction of up to $24,000 to the $24,086 of IRA and pension income, so the remaining income fell within the standard deduction and tax was $0. The reference's only subtraction is the $13,655.95 of taxable Social Security, so WI AGI stays at $29,981.71 and the result is $284.74 after the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"$430 matches the reference's method (subtract taxable Social Security, sliding standard deduction, $950 exemptions, $300 school property tax credit) applied to WI income that includes the $3,753 of capital gains less the 30% long-term exclusion (~$32,609 WI AGI, ~$19,300 taxable). The reference's federal AGI of $43,637.65 contains no capital gains, so its WI AGI is $29,981.71 and its taxable income is $16,360.34." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67 retirement-income subtraction plus a capital-gain subtraction, which left income below the standard deduction and tax at $0. The reference subtracts only the $13,655.95 of taxable Social Security, with no retirement subtraction, giving $16,360.34 taxable and $284.74 after the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It excluded Social Security and applied the standard deduction and a capital-gain adjustment, but included no nonrefundable credit, so $770 is a pre-credit tax. The $300 school property tax credit applies; the reference's tax before that credit is $584.74 on $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It reached $0 by subtracting the IRA and pension income under an age-based Wisconsin retirement-income subtraction before the standard deduction and exemptions. The reference subtracts only the $13,655.95 of taxable Social Security, so WI AGI is $29,981.71, taxable income is $16,360.34, and the result is $284.74 after the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It combined a retirement-income deduction with the standard deduction, exemptions, and credits to reach $0. The reference applies no retirement subtraction; WI AGI of $29,981.71 less $12,671.36 and $950 leaves $16,360.34 taxable, and the $300 school property tax credit covers only part of the $584.74 tax." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"It relied on Wisconsin retirement exclusions plus the standard deduction to bring the tax to $0. The reference's only subtraction is the $13,655.95 of taxable Social Security, giving WI AGI of $29,981.71, $16,360.34 taxable, $584.74 of tax, and $284.74 after the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It used a $366 school property tax credit, which exceeds the $300 cap (12% of the first $2,500 of property tax), and a $10,683 standard deduction, below the single sliding amount (~$12,356 at its own $32,680 WI AGI). Its WI AGI also includes the $3,753 of capital gains, which the reference's $29,981.71 does not." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It followed the reference's steps (subtract taxable Social Security, single sliding standard deduction, $950 exemptions, 3.5%/4.4% rates, $300 school property tax credit) on WI AGI of $32,680, which includes the $3,753 of capital gains less the 30% long-term exclusion. The reference's federal AGI of $43,637.65 contains no capital gains, giving WI AGI of $29,981.71 and $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"Its method matches the reference's (taxable Social Security subtraction, $12,322 phased single standard deduction, $950 exemptions, 3.5%/4.4% rates, $300 school property tax credit), but it works from WI AGI of $32,680, which includes the $3,753 of capital gains less a $1,126 exclusion. The reference's WI AGI of $29,981.71 carries no capital gains, so its taxable income is $16,360.34 rather than $19,408." -us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"It used the pre-2023 Wisconsin rates of 3.54%/4.65%, an unphased $13,900 standard deduction, no $950 of personal and age exemptions, and no $300 school property tax credit. The reference phases the deduction to $12,671.36, subtracts $950, taxes $16,360.34 at 3.5%/4.4%, and subtracts the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It taxed the second bracket at 4.65% instead of 4.4% and used an ~$11,351 standard deduction, below the single sliding-scale amount at its income. Its WI income of ~$32,679 also includes the $3,753 of capital gains, which the reference's $29,981.71 WI AGI does not." -us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It followed the reference's steps (subtract taxable Social Security, single sliding standard deduction phased at 12%, $950 exemptions, 3.5%/4.4% brackets, $300 school property tax credit) on WI income of $32,609, which includes the $3,753 of capital gains less the 30% long-term exclusion. The reference's federal AGI of $43,637.65 contains no capital gains, so its WI AGI is $29,981.71 and its tax is $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It applied a $24,000 retirement-income exclusion to the IRA and pension income, which dropped Wisconsin income to ~$9,806, then used an inflated $24,000 surviving-spouse standard deduction to reach $0. The reference subtracts only the $13,655.95 of taxable Social Security and uses the $12,671.36 single sliding deduction, leaving $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It used a married-joint standard deduction (~$13,300), taxed all $20,506 at 3.5% while ignoring the 4.4% bracket above ~$15,000, and invented an income phase-out that cut the school property tax credit to ~$276. The credit is a flat 12% of the first $2,500 of property taxes ($300), and the reference's single sliding deduction leaves $16,360.34 taxable." -us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"Its arithmetic contradicts itself: it computes $297 of tax and ~$339 of credits but reports $417.88. It also uses a $10,430 head-of-household deduction and invents a 10.82% personal tax credit and a $307 itemized deduction credit on real estate taxes, which do not qualify. The reference uses the $12,671.36 single deduction and only the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,830 matches about $44,700 of Wisconsin taxable income at 3.5%/4.4% with no credit, which requires leaving taxable Social Security in the base and skipping the standard deduction and exemptions. The reference subtracts the $13,655.95 of taxable Social Security, the $12,671.36 deduction, and $950 of exemptions, leaving $16,360.34 taxable." +us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It found the roughly $24,000 retirement income exclusion for 65+/67+ filers and said the standard deduction fully covers the remaining ~$9,700 of Wisconsin income. It still reported $953 instead of carrying that to zero taxable income and $0 tax. It also skipped the 30% capital-gain subtraction on the $3,753 of capital gain distributions." +us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It refused the retirement exclusion because it thought AGI exceeded a limit, mixing up the old $5,000 retirement subtraction's income test with the 67+ exclusion of up to $24,000, which has no income cap and removes $24,000 here. It also used the joint-filer standard deduction scale for a surviving spouse and left out the 30% capital-gain subtraction, ending at $247 instead of $0." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It taxed about $27,500 of Wisconsin income and never applied the $24,000 retirement income exclusion for filers 67 and older, which leaves only about $8,609 of Wisconsin AGI, fully covered by deductions. It also understated the single standard deduction at about $5,000 when it is about $12,461 at this income." +us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It stopped at Wisconsin AGI of $33,735 after subtracting only Social Security. It missed the 30% capital-gain subtraction ($1,125.90) and, more importantly, the $24,000 retirement income exclusion for filers 67 and older, which brings taxable income to zero." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It computed tax on about $19,006 of taxable income after subtracting only Social Security and the standard deduction. It never applied the $24,000 retirement income exclusion for filers 67 and older, which removes the IRA and pension income and leaves no taxable income." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted only Social Security and a flat $13,410 standard deduction, then taxed the rest at outdated 3.54%/4.65% rates. It missed the $24,000 retirement income exclusion for filers 67 and older, which brings Wisconsin taxable income and tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It took the old $5,000 retirement income subtraction instead of the 67+ retirement income exclusion, which removes $24,000 of this filer's $24,086 of pension and IRA income. That error left $15,225 taxable instead of zero. It also dropped the $250 age-65 exemption." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It taxed $24,969 after subtracting only Social Security and an $8,837 standard deduction. It never applied the $24,000 retirement income exclusion for filers 67 and older, the $950 of exemptions, or the 30% capital-gain subtraction. The exclusion alone brings tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $29,446 of Wisconsin taxable income reflects only the Social Security subtraction. It never applied the $24,000 retirement income exclusion for filers 67 and older, which with the standard deduction and exemptions leaves no taxable income and $0 tax." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $1,137 matches taxing Wisconsin AGI after only the standard deduction, without the $24,000 retirement income exclusion for filers 67 and older. That exclusion brings Wisconsin AGI to about $8,609, fully covered by the standard deduction and exemptions, so tax is $0." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted only Social Security and the standard deduction, leaving roughly $20,000+ taxable. It left out the $24,000 retirement income exclusion for filers 67 and older, which removes the IRA and pension income and brings tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reached $25,235 of taxable income after only the Social Security exemption and the sliding standard deduction, then took the $300 school property tax credit. It missed the $24,000 retirement income exclusion for filers 67 and older, under which taxable income is zero." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $1,973 matches taxing most of the ~$33,700 of non-Social-Security income with little deduction. It applied neither the full sliding standard deduction and exemptions nor the $24,000 retirement income exclusion for filers 67 and older, which brings tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It began from Wisconsin AGI of $33,735 after removing only Social Security, then applied the standard deduction and senior exemption. It never subtracted the $24,000 retirement income exclusion for filers 67 and older, which leaves no taxable income." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It submitted $1,348 while its reasoning says $1,617. Both numbers come from taxing income without the $24,000 retirement income exclusion for filers 67 and older, which brings Wisconsin AGI to about $8,609 and tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $1,428 net tax after the school property tax credit means it taxed the IRA and pension income in full. It missed the $24,000 retirement income exclusion for filers 67 and older, which with the standard deduction and exemptions eliminates the tax." +us,scenario_042,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It noted a retirement income subtraction for older filers but did not apply its actual size: $24,000 of this filer's $24,086 of pension and IRA income. Applying it leaves about $8,609 of Wisconsin AGI, fully covered by the standard deduction, so the $123.45 residual has no basis and the tax is $0." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It took out only Social Security and the capital-gain adjustment before applying deductions, and it never applied the $24,000 retirement income exclusion for filers 67 and older, which brings tax to $0. It also missed the $300 school property tax credit available on the standard path." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $430 roughly matches the standard path: Social Security subtraction, standard deduction, $950 of exemptions, and the school property tax credit. It never compared that to the $24,000 retirement income exclusion path for filers 67 and older, which gives $0 and is the one elected." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied only the Social Security exemption, the standard deduction, and the capital-gain adjustment, and it never took the $24,000 retirement income exclusion for filers 67 and older, which brings Wisconsin taxable income to zero. It also left out the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It built the standard path correctly in outline: Social Security and 30% capital-gain subtractions, standard deduction, $950 of exemptions, school property tax credit. It never applied the $24,000 retirement income exclusion for filers 67 and older, which cuts Wisconsin AGI to about $8,609 and tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It got close to the standard-path result ($423 vs. $408.70) but stopped there. The $24,000 retirement income exclusion for filers 67 and older leaves no taxable income, and the filer elects that $0 path." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its $419 is essentially the standard-path tax after the $300 school property tax credit. It left out the $24,000 retirement income exclusion for filers 67 and older, which eliminates the taxable income and gives $0." +us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted only Social Security and a flat $13,900 standard deduction, skipped the exemptions, and used outdated 3.54%/4.65% rates. Most importantly, it never applied the $24,000 retirement income exclusion for filers 67 and older, which brings tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It took the standard path: Social Security subtraction, sliding standard deduction, $950 of exemptions, $300 school property tax credit. It never applied the $24,000 retirement income exclusion for filers 67 and older, which leaves no Wisconsin taxable income." +us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It reached the right Wisconsin AGI of about $32,609 after the Social Security and 30% capital-gain subtractions, then followed only the standard-deduction path to $429.26. It never applied the $24,000 retirement income exclusion for filers 67 and older, which brings taxable income and tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It subtracted Social Security and 30% of capital gains, then applied the standard deduction and exemption, but it never applied the $24,000 retirement income exclusion for filers 67 and older. That exclusion takes out the $19,200 of IRA income and all but $86 of the $4,886 pension, leaving no taxable income." +us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a married-joint standard deduction and a made-up phase-out of the school levy credit. Most importantly, it never applied the $24,000 retirement income exclusion for filers 67 and older, under which Wisconsin taxable income and tax are zero." +us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Its reasoning computed $297 of tax and then subtracted credits, yet it reported $417.88, which is internally inconsistent. It never applied the $24,000 retirement income exclusion for filers 67 and older, which brings tax to $0." +us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $1,830 matches taxing nearly all ~$33,700 of non-Social-Security income with minimal deductions. It missed both the full sliding standard deduction and the $24,000 retirement income exclusion for filers 67 and older, under which tax is $0." us,scenario_042,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model understated Wisconsin homestead-credit household income by excluding substantial Social Security, taxable IRA distributions, pension income, dividends, interest, and capital gains, then applied an assumed 2024 maximum-credit formula instead of the 2026 eligibility computation. The complete computation produces no Wisconsin refundable credit." us,scenario_042,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric state_refundable_credits output or explanation, violating the required structured-output contract." us,scenario_043,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model applied the childless EITC phase-in (7.65% × $4,073) without checking IRC §32(c)(1)(A)(ii)(II), which limits the no-qualifying-children EITC to filers at least 25 and under 65 at the close of the tax year; the head is 66, so the credit is $0 for 2026. Its reasoning tested only the earned-income and AGI phase-out thresholds and never the upper age bar, treating the ARPA-2021 suspension of that bar as if it still applied in 2026. With EITC out, all five refundable components (EITC, refundable AOTC, refundable CTC, Recovery Rebate, refundable payroll tax credit) are $0." @@ -3988,130 +4028,133 @@ us,scenario_045,snap,claude-opus-4.7,llm_error,state_local_rule,False,"It claime us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so nothing substantive can be graded. The correct answer is $0: gross income of $3,022.97 a month is 232% FPL, above both the 130% gross limit and Michigan's 200% BBCE limit." us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly found that $3,023 a month exceeds the 130% FPL gross limit and that the household is not eligible, which means $0. It submitted $7,032 anyway, a value its own explanation does not support. It also never checked Michigan's 200% FPL BBCE limit (which the household also fails), and it overstated the 130% limit as $1,766 when the guideline gives about $1,695 a month." us,scenario_045,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly computed 0.0425 × ($36,276 − $5,900) ≈ $1,291 and then dropped it. Its later guesses about a higher exemption ($6,350 or $7,350) are unfounded, and the $1,229 it submitted does not follow from any of the figures it stated." -us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $5,800 personal exemption instead of Michigan's 2026 amount of $5,900. That left taxable income $100 too high and produced $1,295.23 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied the $14,600 federal standard deduction, which Michigan does not use (Michigan allows only the $5,900 personal exemption), and subtracted the $21,208 ESI premiums from AGI, which equals the full $36,276 in wages. It also treated the Homestead Property Tax Credit as nonrefundable, which zeroed out a tax that is actually $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It used a $5,600 exemption instead of $5,900, which gave about $1,303. It then submitted $1,280, a number its own arithmetic does not support. The correct result is ($36,276 − $5,900) × 4.25% = $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $5,800 personal exemption instead of the 2026 amount of $5,900, so taxable income came out at $30,476 rather than $30,376. That produced $1,295 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning computed $1,295 using a $5,800 exemption (already $100 short of the correct $5,900), but it submitted $1,024. Nothing in its reasoning produces that number." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $5,800 exemption instead of $5,900 and then subtracted about $60 of made-up nonrefundable credits. No Michigan nonrefundable credit applies to this childless filer, so the tax is $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It subtracted a $5,800 personal exemption instead of Michigan's 2026 amount of $5,900. That produced $1,295.23 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used 4.05%, the one-time 2023 reduced rate, instead of the 4.25% rate that applies in 2026. It also projected a $6,000 exemption instead of $5,900, which gave $1,226.18 rather than $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $1,295 with a $5,800 exemption and then raised it to $1,490 by citing an 'exemption phase-in' that does not exist. Michigan's exemption does not phase out at this income, and the correct tax is ($36,276 − $5,900) × 4.25% = $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from wages and arrived at an AGI of $15,068. The actual AGI is the full $36,276 in wages. It also used a $6,200 exemption instead of $5,900, which gave $376.89." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $15,068 by subtracting the $21,208 ESI premiums, but AGI equals the full $36,276 in wages. It also used a $5,670 exemption instead of $5,900, which gave $399 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $5,600 personal exemption, an outdated amount, instead of the 2026 amount of $5,900. That overstated taxable income by $300 and gave $1,303.73." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It projected a $6,000 personal exemption instead of the published 2026 amount of $5,900. That understated taxable income by $100 and gave $1,286.73 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums and took AGI as $15,068, but AGI equals the full $36,276 in wages. It also used a $5,600 exemption instead of $5,900, which gave $402.39." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It showed no derivation. At 4.25%, its $1,150 implies taxable income of about $27,059, which means an exemption of roughly $9,200 instead of $5,900. The correct result is $30,375.59 × 4.25% = $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its taxable income of $30,476 reflects a $5,800 exemption instead of the 2026 amount of $5,900. That produced $1,295 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a 'Federal AGI' of $15,068 by subtracting the $21,208 ESI premiums, but AGI is the full $36,276 in wages. It also used a $5,600 exemption instead of $5,900, which gave $402.39." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed exemptions and 'standard deductions' eliminate the tax, but Michigan has no standard deduction for a 44-year-old filer. Only the $5,900 personal exemption applies, which leaves $30,375.59 taxed at 4.25% for $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used $15,068 as AGI after subtracting the $21,208 ESI premiums, but AGI equals the full $36,276 in wages. It also used a $5,600 exemption instead of $5,900, which gave $402.39." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted a $5,800 personal exemption instead of the 2026 amount of $5,900, which produced $1,295.23 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $5,800 personal exemption instead of Michigan's 2026 amount of $5,900, so taxable income came out at $30,476 and tax at $1,295.23 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 Michigan standard deduction and subtracted the Homestead Property Tax Credit as if it were nonrefundable, but that credit is refundable and does not count toward this output. It then submitted $0, which contradicts its own $121 figure. The only reduction is the $5,900 exemption, which gives $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $5,200 personal exemption instead of the 2026 amount of $5,900. That overstated taxable income by $700 and gave $1,321." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed Michigan's deduction structure leaves no taxable income, but Michigan allows only a $5,900 personal exemption here. That leaves $30,375.59 taxed at 4.25% for $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed made-up standard deductions and nonrefundable credits would fully offset the tax. Michigan has no standard deduction for this filer and no applicable nonrefundable credit, so only the $5,900 exemption applies and the tax is $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It applied the 2023-only 4.05% rate instead of the 4.25% rate for 2026. It also used a $5,800 exemption instead of $5,900, which gave $1,234.28." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used the correct $5,900 exemption but started from an AGI of $15,068 after subtracting the $21,208 ESI premiums. AGI is the full $36,276 in wages, so the tax is $1,290.96, not $390." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed a Michigan standard deduction and exemptions offset the full $36,276. In fact Michigan allows only the $5,900 personal exemption for this filer, which leaves $30,375.59 taxable and $1,290.96 of tax." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It took federal AGI as $15,068 by subtracting the $21,208 ESI premiums, but AGI is the full $36,276 in wages. It also projected a $5,880 exemption instead of $5,900, which gave $390." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It projected a $5,800 personal exemption instead of the 2026 amount of $5,900, which produced $1,295 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $6,000 personal exemption instead of the 2026 amount of $5,900, which understated taxable income by $100 and gave $1,286.73." -us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a stale $5,000 personal exemption instead of the 2026 amount of $5,900. That overstated taxable income by $900 and gave $1,329." -us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It subtracted the Homestead Property Tax Credit (about $1,159) from tax before refundable credits, but that credit is refundable and is excluded from this output. It also used a $6,000 exemption instead of $5,900. The correct pre-refundable tax is $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits. The correct derivation is ($36,275.59 − $5,900) × 4.25% = $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used a $6,000 personal exemption instead of Michigan's 2026 amount of $5,900, which gave $1,286.73 instead of $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It computed about $1,329 using a stale $5,000 exemption, correctly called the homestead credit refundable, and then submitted $0 with no supporting step. The correct tax with the $5,900 exemption is $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used an outdated $5,000 personal exemption instead of the 2026 amount of $5,900, so taxable income came out at $31,276 and tax at $1,329.23." -us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $1,303.73 with a $5,600 exemption ($300 short of the correct $5,900) and then submitted $1,116.01, which nothing in its reasoning supports. The correct result is $1,290.96." -us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a standard deduction and the $5,200 child support payment. Michigan has no standard deduction for this filer, and child support paid is not deductible. Only the $5,900 exemption applies, which leaves $1,290.96 of tax." -us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Wrongly treated the Michigan Homestead Property Tax Credit as limited to homeowners. Renters qualify, with 23% of the $10,560 rent ($2,428.80) counted as property tax, which yields a $760.79 refundable credit." -us,scenario_045,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Looked only at the Michigan EITC, which is correctly $0, and never applied the refundable Homestead Property Tax Credit for renters. That credit is 60% × (23% of $10,560 rent − 3.2% of $36,276 resources) = $760.79." -us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own reasoning computed the homestead credit correctly at about $760.78 (23% of rent, minus 3.2% of $36,276, times 60%). It then submitted $884, which nothing in its derivation supports." -us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Considered only the Michigan EITC and the Home Heating Credit, concluded both were $0, and never applied the renter Homestead Property Tax Credit. It then submitted an unsupported $1,209 instead of the $760.79 homestead credit." -us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Counted only the Michigan EITC, which is $0, and left out the refundable Homestead Property Tax Credit for renters. That credit is worth $760.79 here: 60% × ($2,428.80 − $1,160.82)." -us,scenario_045,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Used the outdated 20% rent-to-property-tax factor ($2,112) instead of 23% ($2,428.80). The rest of its method (3.2% threshold, 60% rate) was correct, so it got $570.70 instead of $760.79." -us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Concluded the homestead credit did not apply because no property tax was listed. It missed that renters claim it on 23% of rent paid ($10,560), which gives a $760.79 refundable credit." -us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Correctly found federal and Michigan EITC to be $0, then made up a $1,089 EITC-based figure. It never applied the renter Homestead Property Tax Credit, which is the only refundable credit here ($760.79)." -us,scenario_045,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and a guessed ~55% rate instead of the 60% rate that applies at $36k of resources. That produced $460 instead of $760.79." -us,scenario_045,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Counted only the zero Michigan EITC and wrongly stated that no other refundable credits apply. The renter Homestead Property Tax Credit applies and equals $760.79. -us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the $21,208 employer-sponsored premiums, used 20% of rent instead of 23%, and added a $102.60 Michigan EITC on a made-up $342 federal EITC. A childless filer with $36k of wages gets no federal EITC, and resources stay at $36,276." -us,scenario_045,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and an 80% credit rate instead of 60%. The correct figure is 0.6 × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and never applied the 60% credit percentage, so it reported the raw $951.17 excess as the credit. The correct figure is 60% × $1,267.98 = $760.79." -us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Applied the 23% rent factor and 60% rate correctly but cut total household resources to $15,068 by subtracting the $21,208 employer-sponsored insurance premiums. Resources stay at the full $36,276, so the threshold is $1,160.82 and the credit is $760.79." -us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Returned $0 and missed the Michigan Homestead Property Tax Credit, which is refundable for renters. With $10,560 rent and $36,276 in resources it equals $760.79." -us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Looked only at the Michigan EITC, which is $0, and left out the refundable renter Homestead Property Tax Credit of 60% × (23% × $10,560 − 3.2% × $36,276) = $760.79." -us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $977.89 homestead figure comes from using 20% of rent instead of 23% and from cutting resources to $15,068 by subtracting the employer premiums. It also added a $62.10 Michigan EITC on a made-up $207 federal EITC that a childless filer at $36k cannot receive." -us,scenario_045,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Michigan Homestead Property Tax Credit for renters, which equals $760.79 for $10,560 rent and $36,276 in resources." -us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Made up a $338.80 federal EITC for a childless filer whose $36,276 in wages is far past the phaseout, so the Michigan EITC is really $0. It also left out the renter Homestead Property Tax Credit of $760.79." -us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $695.48 equals 0.6 × ($2,428.80 − 3.5% × $36,276), so it subtracted 3.5% of household resources instead of 3.2%. The correct subtraction is $1,160.82, which gives $760.79." -us,scenario_045,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076. Child support paid is not a reduction, so resources stay at $36,276, the 3.2% threshold is $1,160.82, and the credit is $760.79." -us,scenario_045,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% to compute $570.70, then submitted $0 anyway, contradicting its own conclusion that the credit applies. The correct credit is 60% × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,Counted only the zero Michigan EITC and wrongly said no other refundable credits apply. It missed the $760.79 renter Homestead Property Tax Credit. -us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no Michigan refundable credit even though $10,560 in rent was listed. Renters qualify for the Homestead Property Tax Credit, worth $760.79 here." -us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,Assumed Michigan refundable credits require a dependent. The Homestead Property Tax Credit has no dependent requirement and pays this single renter $760.79. -us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used the 20% rent factor ($2,112) instead of 23% ($2,428.80), so the 60% credit came to $570.70 instead of $760.79." -us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the $21,208 employer premiums, which inflated the homestead credit to $1,168. It also added a $104 state EITC even though the childless filer's federal EITC is $0." -us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,Assumed a childless household has no Michigan refundable credit. The renter Homestead Property Tax Credit applies without children and equals $760.79. -us,scenario_045,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Treated the $21,208 employer-sponsored insurance premiums as a reduction to household resources, giving $15,068. Resources stay at $36,276, so the 3.2% threshold is $1,160.82 and the credit is $760.79, not $1,167.97." -us,scenario_045,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and never applied the 60% credit percentage, so it submitted the raw $951 excess. The correct credit is 60% × $1,267.98 = $760.79." -us,scenario_045,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from household resources, giving $31,076. That shrank the 3.2% threshold to $994.43 and inflated the credit to $860.62, when resources of $36,276 give $760.79." -us,scenario_045,state_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the employer insurance premiums and used 20% of rent instead of 23%, giving $977.89. The correct calculation is 60% × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Michigan Homestead Property Tax Credit for renters, which equals $760.79 for this household." -us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Cut resources to $15,068 by subtracting the employer premiums and used a 3.5% threshold instead of 3.2%, giving a $1,141 homestead credit. It also added a $20 EITC at the outdated 6% match on a made-up $341 federal EITC, when the childless filer's federal EITC is $0." -us,scenario_045,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076, so the threshold came out too low. With resources of $36,276 the credit is $760.79, not $861." -us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Homestead Property Tax Credit available to Michigan renters, worth $760.79 here." -us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Wrongly treated the Homestead Property Tax Credit as limited to tax liability. It is fully refundable, so the whole $760.79 counts as a state refundable credit. It also used the outdated 6% EITC match." -us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer or explanation for state_refundable_credits, so nothing could be scored against the $760.79 homestead credit." -us,scenario_045,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and a 3.5% resource threshold instead of 3.2%, giving $505.40. The correct figure is 0.6 × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Wrongly treated the listed pre-subsidy rent as not actually paid and decided the renter credit was not triggered. With no subsidy listed, the full $10,560 is rent paid, and 23% of it exceeds 3.2% of resources, producing a $760.79 credit." -us,scenario_045,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Used 17% of rent instead of 23% and a 3.5% threshold instead of 3.2%, giving $315.32. The correct figure is 60% × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Wrongly decided the listed facts do not qualify for the homestead credit. A renter paying $10,560 with $36,276 in resources gets 60% of the amount by which 23% of rent exceeds 3.2% of resources, which is $760.79." -us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Returned $0 and missed the refundable Michigan Homestead Property Tax Credit for renters, which equals $760.79 for this household." +us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly computed 0.0425 × ($36,276 − $5,900) ≈ $1,291, then dropped that result for guesses at higher exemptions. It submitted $1,229, a value that matches none of the exemptions it named. The submitted value contradicts its own correct derivation." +us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used $5,800, which is the 2025 Michigan personal exemption, instead of the 2026 amount of $5,900. That overstates taxable income by $100 and gives $1,295.23 instead of 4.25% × $30,375.59 = $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model applied a federal-style $14,600 standard deduction that Michigan does not have. It subtracted the $21,208 employer insurance premiums from the stated gross wages, and it treated the refundable Homestead Property Tax Credit as nonrefundable to reach $0. The correct base is AGI of $36,275.59 minus the $5,900 personal exemption, taxed at 4.25%." +us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model used a $5,600 exemption, which is the 2024 amount, instead of the 2026 amount of $5,900, and got about $1,303. It then submitted $1,280 with no derivation for that number." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied the 2025 personal exemption of $5,800 instead of the 2026 exemption of $5,900. That produces taxable income of $30,476 and tax of $1,295 instead of $30,375.59 × 4.25% = $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed $1,295 with a $5,800 exemption, itself $100 too low against the 2026 amount of $5,900. It then submitted $1,024, which has no connection to any step in its own reasoning." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $5,800 instead of $5,900 for 2026. It then subtracted about $60 of nonrefundable credits that do not exist for this filer and landed at $1,235. No Michigan nonrefundable credit applies, so the tax is $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model subtracted a $5,800 personal exemption, the 2025 figure, instead of the 2026 exemption of $5,900. It got $1,295.23 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied 4.05%, a rate that was in effect only for 2023, instead of the 2026 rate of 4.25%. It also used a guessed $6,000 exemption instead of $5,900. Both errors together produced $1,226.18." +us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model computed $1,295 using the outdated $5,800 exemption. It then inflated that to $1,490 by citing an exemption phase-in that does not exist. Michigan's $5,900 personal exemption is not reduced at this income, so the tax is $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 employer insurance premiums from the stated gross wages and used $15,068 as AGI, but the correct AGI is $36,275.59. It also used a $6,200 exemption instead of $5,900, which gave $376.89." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $15,068 by subtracting employer insurance premiums from the stated gross wages, when AGI equals the full $36,275.59 of wages. It also used a $5,670 exemption instead of the 2026 amount of $5,900, which gave $399." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the 2024 exemption of $5,600 instead of the 2026 exemption of $5,900. That overstates taxable income by $300 and gives $1,303.73 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a guessed $6,000 personal exemption instead of the published 2026 amount of $5,900. That understates taxable income by $100 and gives $1,286.73 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used Michigan AGI of $15,068 by subtracting the $21,208 employer insurance premiums from the stated gross wages, but AGI is the full $36,275.59. It also used the outdated $5,600 exemption instead of $5,900." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no derivation. Its $1,150 at 4.25% implies taxable income of about $27,059, which means it subtracted roughly $9,200 from $36,276. The correct calculation subtracts only the single $5,900 personal exemption, leaving $30,375.59 and tax of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its taxable income of $30,476 means it subtracted the 2025 exemption of $5,800 instead of the 2026 exemption of $5,900. That gives $1,295 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $15,068 as AGI after subtracting employer insurance premiums from the stated gross wages, when AGI is the full $36,275.59. It also used the outdated $5,600 exemption instead of $5,900, which gave $402.39." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model zeroed out the tax with a standard deduction that Michigan's income tax does not have. The only reduction is the $5,900 personal exemption, which leaves $30,375.59 taxable and $1,290.96 of tax." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $21,208 employer insurance premiums as excluded from the stated gross wages and set AGI at $15,068 instead of $36,275.59. It also used the outdated $5,600 exemption instead of $5,900." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model subtracted the 2025 personal exemption of $5,800 instead of the 2026 exemption of $5,900. It got $1,295.23 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the $5,800 exemption, which is the 2025 figure, rather than the 2026 exemption of $5,900. That overstates taxable income by $100 and gives $1,295.23." +us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a $20,000 Michigan standard deduction and subtracted the refundable Homestead Property Tax Credit as if it were nonrefundable. It then submitted $0, which contradicts even its own $121 result. The correct calculation is 4.25% × ($36,275.59 − $5,900)." +us,scenario_045,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used an outdated personal exemption of about $5,200 instead of the 2026 exemption of $5,900. That overstates taxable income by $700 and gives $1,321." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed a state deduction structure that removes all taxable income at this level. Michigan allows only the $5,900 personal exemption, which leaves $30,375.59 taxable at 4.25%." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed standard deductions and nonrefundable credits wipe out the Michigan tax. Michigan has no standard deduction, and no nonrefundable credit applies. Only the $5,900 exemption is subtracted, leaving $1,290.96 of tax." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied 4.05%, a rate in effect only for 2023, instead of the 2026 rate of 4.25%. It also used the 2025 exemption of $5,800 instead of $5,900, which gave $1,234.28." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the correct $5,900 exemption but started from a federal AGI of $15,068, having subtracted the $21,208 employer insurance premiums from the stated gross wages. AGI is the full $36,275.59, so the tax is $1,290.96, not $390." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed a Michigan standard deduction and exemptions offset all $36,276 of income. Michigan has no standard deduction, and the single $5,900 personal exemption leaves $30,375.59 taxable at 4.25%." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used federal AGI of $15,068 by subtracting employer insurance premiums from the stated gross wages instead of using the full $36,275.59. It also projected a $5,880 exemption instead of $5,900, which gave $390." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model projected the 2026 exemption at $5,800, which is the 2025 value, instead of the published $5,900. It got $1,295 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model subtracted a $6,000 exemption instead of the 2026 amount of $5,900. That understates taxable income by $100 and gives $1,286.73." +us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an outdated $5,000 personal exemption instead of the 2026 exemption of $5,900. That overstates taxable income by $900 and gives $1,329." +us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model subtracted about $1,159 for the Homestead Property Tax Credit, but that credit is refundable and does not belong in tax before refundable credits. It also used a $6,000 exemption instead of $5,900. The correct pre-refundable tax is $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model allowed a $6,000 personal exemption instead of the published 2026 amount of $5,900. It got $1,286.73 instead of 4.25% × $30,375.59 = $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model computed about $1,329 using an outdated $5,000 exemption. It acknowledged that the homestead credit is refundable and excluded here, yet it still submitted $0. The correct figure with the $5,900 exemption is $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model used the 2024 exemption of $5,600 instead of the 2026 exemption of $5,900. That overstates taxable income by $300 and gives $1,303.70." +us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model subtracted an outdated $5,000 personal exemption instead of the 2026 amount of $5,900. It got $1,329.23 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model computed $1,303.73 with the outdated $5,600 exemption, when the 2026 exemption is $5,900. It then submitted $1,116.01, a value no step in its reasoning supports." +us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted a nonexistent Michigan standard deduction and the $5,200 of child support paid, which is not deductible, to reach $0. The correct calculation subtracts only the $5,900 personal exemption, which gives $1,290.96." +us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model ruled out the Homestead Property Tax Credit because the household rents. Michigan counts 23% of rent as property tax for renters, which gives a $760.79 credit here." +us,scenario_045,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model only checked the Michigan EITC, which is correctly $0. It never considered the refundable Homestead Property Tax Credit available to renters: 60% × (23% × $10,560 − 3.2% × $36,276) = $760.79." +us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's reasoning reached the right result: 60% × ($2,428.80 − $1,160.83) = $760.78. It then submitted an unrelated $884 by 'adjusting for slight rounding', so its final answer contradicts its own correct calculation." +us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model looked only at the EITC and the Home Heating Credit, found both to be $0, and never computed the renter's Homestead Property Tax Credit. Its $1,209 is an unsupported number that contradicts its own conclusion that the credits are 'minimal'." +us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model counted only the Michigan EITC, which is $0, and left out the refundable Homestead Property Tax Credit. That credit is worth $760.79 for this renter: 23% of rent, less 3.2% of household resources, times 60%." +us,scenario_045,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model treated 20% of rent as property tax instead of the 23% PolicyEngine uses ($2,428.80). That understated the credit at $570.70 instead of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model dropped the Homestead Property Tax Credit because no property tax was listed. For renters, 23% of the $10,560 rent counts as property tax, which yields a $760.79 refundable credit." +us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model correctly found federal and Michigan EITC to be $0. It then made up a $1,089 'match' figure and never calculated the Homestead Property Tax Credit (60% × ($2,428.80 − $1,160.82) = $760.79), which is the only refundable credit here." +us,scenario_045,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and an assumed ~55% rate instead of 60%. With the correct parameters, $2,428.80 − $1,160.82 = $1,267.98, and 60% of that is $760.79." +us,scenario_045,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model counted only the Michigan EITC, which is $0, and missed the refundable Homestead Property Tax Credit for renters. That credit is $760.79 here." +us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 ESI premiums, cutting household resources to $15,068 instead of $36,275.59, and used 20% of rent instead of 23%. On the same reduced-earnings basis it also produced a $342 federal EITC, although a childless filer with $36,276 in wages gets none." +us,scenario_045,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used 20% of rent instead of 23%, a 3.5% threshold instead of 3.2%, and an 80% rate instead of 60%. The correct figure is 0.60 × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used 20% of rent instead of 23% and never applied the 60% credit percentage. Its answer is the raw excess, $2,112 − $1,160.83, rather than 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied the 23% rent share and the 60% rate correctly, but it cut household resources to $15,068 by subtracting the $21,208 ESI premiums. PolicyEngine counts full wages of $36,275.59, so the 3.2% threshold is $1,160.82, not $482.18." +us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no Michigan refundable credit applies and overlooked the renter's Homestead Property Tax Credit: 60% × (23% × $10,560 − 3.2% × $36,276) = $760.79." +us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model counted only the Michigan EITC, which is $0, and left out the $760.79 refundable Homestead Property Tax Credit that applies to this renter." +us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's $977.89 homestead figure comes from 20% of rent instead of 23%, with household resources cut to $15,068 by subtracting ESI premiums. It also added a Michigan EITC on a $207 federal EITC that doesn't exist for a childless filer with $36,276 in wages." +us,scenario_045,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model reported zero refundable credits and missed the renter's Homestead Property Tax Credit: 23% of rent ($2,428.80), less 3.2% of $36,275.59, times 60% = $760.79." +us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model reported only a Michigan EITC built on a $338.80 federal EITC, which is $0 for this childless filer with $36,276 in wages. It left out the $760.79 Homestead Property Tax Credit, which is the entire answer." +us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $695.48 result equals 60% × ($2,428.80 − 3.5% × $36,276), so the model used a 3.5% resource threshold instead of 3.2%. The correct subtraction is $1,160.82, which gives $760.79." +us,scenario_045,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,200 of child support paid from total household resources, giving $31,076. PolicyEngine uses $36,275.59 with no deduction for child support paid, so 3.2% of resources is $1,160.82 and the credit is $760.79." +us,scenario_045,state_refundable_credits,glm-5.2,llm_error,other,False,"The model's reasoning computed a $570.70 homestead credit, though it used 20% of rent instead of 23%. It then submitted $0, contradicting its own finding that the credit is refundable and applies." +us,scenario_045,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model counted only the Michigan EITC, which is $0, and wrongly said no other refundable credit applies. The renter's Homestead Property Tax Credit is $760.79." +us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no refundable Michigan credit and overlooked the Homestead Property Tax Credit, which is triggered by the $10,560 rent: 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required a dependent for a refundable credit. Michigan's Homestead Property Tax Credit needs no dependent, and this renter qualifies for $760.79." +us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model treated 20% of rent as property tax instead of the 23% PolicyEngine applies. Using $2,428.80 instead of $2,112 raises the credit from $570.70 to $760.79." +us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model cut household resources to $15,068 by subtracting ESI premiums, which inflated the homestead credit to $1,168. It also added a $104 Michigan EITC, although the federal EITC is $0 at $36,276 in wages with no children." +us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model assumed a childless household gets no Michigan refundable credit. The Homestead Property Tax Credit doesn't depend on children and is $760.79 for this renter. +us,scenario_045,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model deducted the $21,208 insurance premiums from household resources, leaving $15,068. PolicyEngine counts $36,275.59, so the 3.2% threshold is $1,160.82 and the credit is $760.79, not $1,167.97." +us,scenario_045,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model used 20% of rent instead of 23% and stopped at the excess over 3.2% of resources without applying the 60% credit percentage. The correct figure is 0.60 × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced household resources by the $5,200 of child support paid, giving $31,076. PolicyEngine counts the full $36,275.59, so the 3.2% subtraction is $1,160.82 and the credit is $760.79." +us,scenario_045,state_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 insurance premiums, cutting resources to $15,068, and used 20% of rent instead of 23%. The correct figure is 60% × ($2,428.80 − 3.2% × $36,275.59) = $760.79." +us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said no state refundable credit applies and missed the renter's Homestead Property Tax Credit, which comes to $760.79." +us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model cut household resources to $15,068 by subtracting ESI premiums and used a 3.5% threshold instead of 3.2%. It also added a Michigan EITC at the outdated 6% rate on a $341 federal EITC that doesn't exist for this childless filer." +us,scenario_045,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,200 of child support paid from household resources, giving $31,076. PolicyEngine counts $36,275.59, so the excess is $1,267.98 and 60% of it is $760.79, not $861." +us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model reported no refundable credits and missed the Homestead Property Tax Credit, which Michigan grants renters: 60% × (23% × $10,560 − 3.2% × $36,276) = $760.79." +us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model treated the Homestead Property Tax Credit as limited to tax liability. It is fully refundable, so the whole $760.79 is a refundable credit. The model also cited the outdated 6% Michigan EITC rate." +us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model gave no value or explanation for state_refundable_credits. The required $760.79 Homestead Property Tax Credit is absent. +us,scenario_045,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used 20% of rent instead of 23% and a 3.5% resource threshold instead of 3.2%. The correct figure is 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model wrongly concluded that no rent was actually paid and that the homestead credit isn't triggered. The $10,560 rent is paid rent, 23% of it ($2,428.80) is far above 3.2% of resources, and the credit is $760.79." +us,scenario_045,state_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model used a 3.5% household-resource threshold ($1,269.66) instead of 3.2% ($1,160.82). With 3.2%, the excess is $1,267.98 and the credit is $760.79." +us,scenario_045,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model treated only 17% of rent as property tax, instead of 23%, and used a 3.5% resource threshold instead of 3.2%. That shrank the credit to $315.32 instead of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model concluded the homestead credit doesn't qualify on the listed facts. The listed $10,560 rent makes the renter eligible, and the credit is $760.79." +us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model reported no refundable state credits and missed the renter's Homestead Property Tax Credit: 23% of rent, less 3.2% of $36,275.59 in resources, times 60% = $760.79." us,scenario_046,child1_chip_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied a generic 255% FPL CHIP limit that is not Oklahoma's rule. In Oklahoma a 10-year-old falls outside the separate CHIP's age coverage and gets children's coverage only through SoonerCare Medicaid, whose income test the household also fails. It also understated MAGI as $77,164 by subtracting the $14,839 FLSA overtime premium ($92,003 - $14,839), but the overtime deduction does not reduce AGI-based MAGI. Correct MAGI is about $92,003, well above Oklahoma's children's limit, so the answer is not eligible." us,scenario_046,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model asserted that the child was “likely eligible” without testing Oklahoma's income limits or any categorical pathway. At 2.79 times FPL, child1 fails the regular children's Medicaid limit, CHIP limit, and all other applicable categories, yielding medicaid_category = NONE and value = 0." us,scenario_046,child2_chip_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model's MAGI of $77,164 is exactly $92,003 minus the $14,839 FLSA overtime premium. It treated the no-tax-on-overtime deduction as lowering MAGI, but that deduction is taken below the line and never reduces AGI or Medicaid/CHIP MAGI. Using the correct MAGI of $92,003, the four-person household sits at about 280% FPL. That is above even the 255% CHIP limit the model cited, so child2 is not CHIP-eligible." us,scenario_046,child2_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed that being a child made Child 2 eligible without testing Oklahoma's categorical and income requirements. At 2.79 times FPL, Child 2 exceeds all applicable child Medicaid limits and qualifies through no other pathway, so the correct value is 0." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own derivation was nearly right: taxable income of $44,864, tax of $4,888 less the $4,400 CTC, about $488. It then submitted $3,465 based on an unexplained 'adjustment', dropping its correct result. It also used a $32,300 standard deduction instead of $32,200." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction and used a stale $28,700 MFJ standard deduction instead of $32,200, which inflated taxable income to $63,303. It also applied a $2,500 AOTC even though there were no qualified tuition expenses. It used a $2,000-per-child CTC instead of $2,200 and capped the CTC incoherently." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed without any computation that credits wipe out the tax. Taxable income is $44,964.29 after the $32,200 standard deduction and the $14,838.71 overtime deduction, which gives $4,899.71 of tax. That exceeds the $4,400 CTC, so $499.71 remains." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never applied the OBBBA qualified overtime deduction ($14,838.71) and used a $31,500 standard deduction instead of $32,200, which gave $60,503 of taxable income. It used a $2,000-per-child CTC instead of $2,200, reached $2,777, and then submitted an unrelated $4,669." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction, leaving $59,703 of taxable income. It then submitted $6,779, roughly its pre-credit tax, without subtracting the nonrefundable CTC, even though its own reasoning said credits bring the tax near zero." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It correctly reached $44,964 of taxable income but used a $24,000 top for the 10% bracket instead of the 2026 MFJ $24,800. It then reported pre-credit tax ($4,966) instead of subtracting the $4,400 nonrefundable CTC, which the output definition requires." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the OBBBA qualified overtime deduction of $14,838.71. It used 2025 parameters instead of 2026 ones: a $30,000 standard deduction instead of $32,200, a $23,850 top for the 10% bracket instead of $24,800, and a $2,000-per-child CTC instead of $2,200. This produced $62,003 of taxable income and $2,963 of tax." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the FLSA overtime premium as purely informational and so never took the $14,838.71 qualified overtime deduction. It then submitted $12,200, which exceeds even its own $6,696 pre-credit tax, instead of subtracting the $4,400 nonrefundable CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It itemized at about $40.9k by double-counting the $12,000 health insurance premium, which is one amount reported under two labels. Correct itemized deductions are about $28.7k, below the $32,200 standard deduction. Taxable income is therefore $44,964.29, tax is $4,899.71, and $499.71 remains after the CTC instead of $0." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It omitted the qualified overtime deduction and double-counted premiums as $24,330 of medical costs, so it wrongly itemized. It also treated only $200 per child of the CTC as nonrefundable. In fact the whole $2,200-per-child CTC is nonrefundable up to the tax liability, and only the excess is refundable as the ACTC." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law: personal exemptions, a 15% second bracket and a $1,000-per-child CTC. OBBBA made the TCJA rates, the $32,200 standard deduction and the $2,200 CTC permanent. It also omitted the $14,838.71 qualified overtime deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed under expired pre-TCJA rules, with personal exemptions and a $1,000-per-child CTC, instead of OBBBA's 2026 regime: the $32,200 standard deduction, 10%/12% brackets and a $2,200 CTC. It also skipped the $14,838.71 qualified overtime deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its taxable income of $44,964 was right, but it ended the 10% bracket at $24,400 instead of the 2026 MFJ $24,800. It also used an inflated $2,260-per-child CTC instead of $2,200, which netted $387.68 instead of $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It added the overtime premium on top of wages even though gross wages already include overtime. It subtracted ESI premiums from AGI, which no input supports, and used pre-TCJA personal exemptions and a $1,000-per-child CTC. It also never applied the $14,838.71 overtime deduction or the $32,200 standard deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It omitted the qualified overtime deduction, used a $31,200 standard deduction and a $4,000 CTC, and submitted $8,518. That figure is larger than any tax its stated inputs produce. The correct taxable income is $44,964.29, giving $4,899.71 of tax less $4,400 of CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law (personal exemptions and a $1,000-per-child CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It subtracted ESI premiums from AGI with no basis, and omitted the $14,838.71 qualified overtime deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used pre-TCJA personal exemptions, a $1,000-per-child CTC and an ESI-premium subtraction from AGI. It also claimed a $1,500 nonrefundable AOTC with no qualified tuition expenses, and omitted the $14,838.71 overtime deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,212 is consistent with taxing AGI less only a standard deduction and omitting the $14,838.71 qualified overtime deduction. The correct path gives $44,964.29 of taxable income, $4,899.71 of tax, and $499.71 after the $4,400 CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no usable derivation. $1,858.40 does not reflect the correct path: the $32,200 standard deduction plus the $14,838.71 overtime deduction, giving $44,964.29 of taxable income and $4,899.71 of tax. From that, the full $4,400 nonrefundable CTC leaves $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its reference to personal exemptions shows it applied pre-TCJA sunset rules instead of OBBBA's permanent 2026 regime of a $32,200 standard deduction and a $2,200 CTC. It also omitted the $14,838.71 qualified overtime deduction, producing $2,722.40." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction. It used 2025 values instead of 2026 ones: a $30,000 standard deduction instead of $32,200, a $23,850 top for the 10% bracket instead of $24,800, and a $2,000-per-child CTC instead of $2,200. The result was $2,963.36." -us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No answer or reasoning was returned, so there is no parseable value for this output." -us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It computed $4,899.68 of pre-credit tax correctly, then treated only $500 per child ($2,200 minus the $1,700 ACTC cap) as nonrefundable. The full $4,400 CTC is nonrefundable up to the tax liability, and the ACTC applies only to the unused excess. The answer should have been $499.71, not $3,899.68." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It assumed the American Opportunity Credit offsets the tax even though no qualified tuition expenses are listed, so AOTC is $0. It never computed a liability. The correct result is $4,899.71 of tax less the $4,400 CTC, or $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction and applied an AOTC with no qualified expenses. It also did not subtract the $4,400 nonrefundable CTC, and reported $6,794, which is roughly the tax with no overtime deduction and no CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It capped the qualified overtime deduction at $12,500, the single-filer limit, instead of the $25,000 joint-return limit. That cut the deduction from $14,838.71 and left $47,303 of taxable income instead of $44,964.29." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It invoked an irrelevant AOTC phaseout, omitted the $14,838.71 overtime deduction, and failed to subtract the $4,400 nonrefundable CTC, reporting an unsupported $8,500. The correct result is $4,899.71 of tax on $44,964.29 of taxable income, less $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law (personal exemptions, a 15% bracket and a $1,000-per-child CTC) instead of OBBBA's permanent 2026 rules. It also omitted the $14,838.71 overtime deduction and itemized instead of taking the larger $32,200 standard deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunset, using personal exemptions and $1,000-per-child CTCs, even though OBBBA kept the $32,200 standard deduction, the 10%/12% brackets and the $2,200 CTC for 2026. It also never applied the $14,838.71 qualified overtime deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It computed a 'post-TCJA' return with personal exemptions and a $1,000-per-child CTC instead of OBBBA's permanent 2026 parameters. It also omitted the $14,838.71 qualified overtime deduction." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used pre-TCJA rules, with personal exemptions and a $2,000 total CTC, and reached $23,508 of taxable income. The correct figures are $44,964.29 of taxable income after the $32,200 standard deduction and the $14,838.71 overtime deduction, with a $4,400 CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction correctly but inflated the medical deduction to about $19,193 by double-counting the $12,000 premium, which is one amount under two labels. That made itemizing appear to beat the $32,200 standard deduction. With the standard deduction, tax is $4,899.71, and $499.71 remains after the $4,400 CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or reasoning was returned, so there is no parseable value for this output." -us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted a $9,999 placeholder with no derivation. The correct result is $499.71: the standard and overtime deductions give $44,964.29 of taxable income and $4,899.71 of tax, less the $4,400 CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted medical expenses as $24,980 by double-counting the $12,000 premium and including over-the-counter costs, which produced $40,772 of itemized deductions. Correct itemized deductions fall below the $32,200 standard deduction. Taxable income is therefore $44,964.29 and tax is $4,899.71, which the $4,400 CTC does not fully offset." -us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified overtime deduction and double-counted the $12,000 premium, plus OTC costs, to justify itemizing. It also claimed a $1,000 AOTC despite no qualified expenses and used a $2,000-per-child CTC instead of $2,200." -us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a full $2,500 AOTC even though no qualified tuition expenses exist, and omitted the $14,838.71 overtime deduction. It then submitted $8,838, which contradicts its own $342 result." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working was nearly right: the overtime deduction, standard deduction, taxable income of about $44,864 and tax of about $4,888 less the $4,400 CTC gave about $488. It then submitted $3,465 with no derivation. Correct inputs ($32,200 standard deduction, $24,800 top of the 10% bracket) give $4,899.71 − $4,400 = $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the $14,839 qualified overtime deduction and used an outdated $28,700 standard deduction instead of $32,200, which inflated taxable income to $63,303 instead of $44,964. It also applied a $2,500 AOTC even though no qualified tuition expenses were listed, and it used $2,000 per child for the CTC instead of $2,200." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the credits wipe out the liability without computing it. After the $32,200 standard deduction and the $14,839 overtime deduction, taxable income is $44,964 and tax is $4,899.71. That exceeds the $4,400 nonrefundable CTC, so $499.71 remains, not $0." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never applied the $14,839 qualified overtime deduction and used the 2025 $31,500 standard deduction instead of $32,200, which gave taxable income of $60,503. It also used a $2,000-per-child CTC instead of $2,200. Its submitted $4,669 does not even match its own $2,777 result." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It left out the $14,839 qualified overtime deduction, so taxable income came out at $59,703 instead of $44,964. Although it said the CTC would bring tax near zero, the $6,779 it submitted is the pre-credit tax with no CTC subtracted." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It reached the correct taxable income of $44,964 but used a $24,000 top for the 10% bracket instead of $24,800. More importantly, it reported the pre-credit tax ($4,966) as the answer instead of subtracting the $4,400 nonrefundable CTC. The output is defined as tax after nonrefundable credits, so the answer is $4,899.71 − $4,400 = $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used the 2025 parameters (a $30,000 standard deduction and a $23,850 top for the 10% bracket) instead of the 2026 values of $32,200 and $24,800. It never applied the $14,839 qualified overtime deduction, so taxable income was $62,003 instead of $44,964. It also used a $2,000-per-child CTC instead of $2,200." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It wrongly treated the FLSA overtime premium as purely informational and never took the $14,839 qualified overtime deduction, which inflated taxable income to $59,803. It then submitted $12,200, a figure unrelated to its own roughly $6,696 pre-credit tax, instead of subtracting the $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It itemized about $40.9k by counting the duplicated $12,000 premium fields twice and including non-deductible OTC costs. Allowable itemized deductions are only about $28.8k: $5,430 of medical above the 7.5% floor, $12,415 of mortgage interest, and SALT. That is less than the $32,200 standard deduction, so taxable income is $44,964 and tax of $4,899.71 exceeds the $4,400 CTC, which leaves $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It never applied the $14,839 qualified overtime deduction and itemized $39,845 by double-counting the $12,000 premium fields, when the standard deduction is the better choice. It also wrongly limited the nonrefundable CTC to $400 by subtracting the refundable ACTC cap from each child's credit. In fact the full $4,400 CTC is nonrefundable up to the tax liability." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law for 2026, with personal exemptions, a 15% bracket and a $1,000-per-child CTC, but OBBBA made the TCJA structure permanent. The correct calculation uses the $32,200 standard deduction, the $14,839 overtime deduction, 10%/12% brackets and a $2,200-per-child CTC, giving $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, so it used personal exemptions, pre-TCJA brackets and a $1,000-per-child CTC. For 2026 the law is the $32,200 standard deduction plus the $14,839 overtime deduction, giving taxable income of $44,964 and a $4,400 CTC. It also never applied the overtime deduction." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its taxable income of $44,964 was correct, but it used $24,400 instead of $24,800 as the top of the 2026 MFJ 10% bracket ($4,907.68 instead of $4,899.71). It also inflated the CTC to $2,260 per child instead of the statutory $2,200, which overstated the nonrefundable credit by $120." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It added the $14,839 overtime premium on top of wages, even though the prompt says wages already include overtime. It then subtracted the ESI premiums from gross wages and applied pre-TCJA personal exemptions with a $1,000-per-child CTC. Correct AGI is $92,003, and the overtime premium should be a deduction, not extra income." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It never applied the $14,839 qualified overtime deduction and used a $31,200 standard deduction and a $4,000 CTC. Its $8,518 exceeds even the pre-credit tax those inputs produce (about $6,700), so the number is internally inconsistent. The correct result is $4,899.71 − $4,400 = $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It wrongly subtracted the $9,208 ESI premiums from the stated gross wages and used post-TCJA-sunset rules: personal exemptions and a $1,000-per-child CTC. It also never applied the overtime deduction. Under OBBBA for 2026, the $32,200 standard deduction and the $14,839 overtime deduction give taxable income of $44,964 and a $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It subtracted the ESI premiums from gross wages, used pre-TCJA personal exemptions with a $1,000-per-child CTC, and skipped the overtime deduction. It also claimed a $1,500 nonrefundable AOTC even though no qualified tuition expenses were listed, so the AOTC is $0." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $44,964 of taxable income (after the $32,200 standard deduction and $14,839 overtime deduction), $4,899.71 of tax, less the $4,400 CTC, for $499.71. The $3,212 figure matches taxing roughly $60k+ of income without the overtime deduction and netting only a $4,000 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $1,858.40 matches the post-TCJA-sunset framework that other Gemini answers used: itemized deductions plus personal exemptions, pre-TCJA brackets and a $2,000 total CTC. Correct 2026 law uses the $32,200 standard deduction, the $14,839 overtime deduction and a $4,400 CTC, giving $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied personal exemptions, which means it assumed the TCJA expired in 2026, and it never took the qualified overtime deduction. Under OBBBA, exemptions are zero and the household gets the $32,200 standard deduction, the $14,839 overtime deduction and a $2,200-per-child CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and the 2025 $23,850 10% bracket instead of the 2026 values of $32,200 and $24,800. It left out the $14,839 qualified overtime deduction, so taxable income was $62,003 instead of $44,964. It also used $2,000 per child for the CTC instead of $2,200." +us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no parseable value or explanation for this output. The correct result is $4,899.71 of tax on $44,964.29 of taxable income, less the $4,400 nonrefundable CTC, for $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"Its taxable income ($44,964) and tax ($4,899.68) were correct, but it limited the nonrefundable CTC to 2 × ($2,200 − $1,700) = $1,000. The full $2,200-per-child CTC is nonrefundable up to the tax liability, and the $1,700 ACTC cap only governs refunds of the unused part. All $4,400 applies, which leaves $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It refused to compute and defaulted to $0, citing an AOTC even though no qualified tuition expenses were listed, so the AOTC is $0. The correct result is taxable income of $44,964 after the standard and overtime deductions, $4,899.71 of tax, less the $4,400 CTC, for $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $6,794 matches the pre-credit tax on roughly $60k of taxable income. That means it left out the $14,839 qualified overtime deduction and never subtracted the $4,400 nonrefundable CTC. It also cited an AOTC even though there were no qualified expenses." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It capped the qualified overtime deduction at $12,500, which is the cap for single filers. The married-filing-jointly cap is $25,000, so the full $14,839 premium is deductible, giving taxable income of $44,964 instead of $47,303 and tax of $4,899.71 before the $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave a round $8,500 with no computation. It invoked an AOTC phase-out even though AGI is far below the threshold, and it never applied the overtime deduction or subtracted the $4,400 CTC. The correct result is $4,899.71 − $4,400 = $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA law with four personal exemptions, a 15% bracket and a $1,000-per-child CTC, but OBBBA made the TCJA structure permanent for 2026. It also never took the $14,839 overtime deduction and itemized when the $32,200 standard deduction was larger." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, so it used personal exemptions, pre-TCJA brackets and a $1,000-per-child CTC. It also never applied the qualified overtime deduction. Under 2026 law the household takes the $32,200 standard deduction and the $14,839 overtime deduction and claims a $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used a 'post-TCJA' 2026 return with $21,200 of personal exemptions and a $1,000-per-child CTC, but OBBBA made the TCJA standard deduction and CTC permanent. It also left out the $14,839 overtime deduction, so taxable income was $41,919 instead of $44,964 and its tax was computed under the wrong regime." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules with a $2,000 total CTC and arrived at only $23,508 of taxable income. The correct calculation is the $32,200 standard deduction plus the $14,839 overtime deduction, giving taxable income of $44,964 and tax of $4,899.71, less the $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized with about $19,193 of medical deductions by counting both $12,000 premium fields and the non-deductible OTC expenses. Allowable medical is only about $5,430, so itemized deductions (about $28.8k) fall below the $32,200 standard deduction. Taxable income is $44,964, and tax of $4,899.71 exceeds the $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no parseable value or explanation for this output. The correct result is $4,899.71 of tax on $44,964.29 of taxable income, less the $4,400 nonrefundable CTC, for $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted a $9,999 placeholder with no computation. The correct derivation is the $32,200 standard deduction plus the $14,839 overtime deduction, giving $44,964 of taxable income and $4,899.71 of tax, less the $4,400 CTC, for $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Its taxable income of $44,964 and $4,400 CTC were correct, but it approximated tentative tax at $4,910. Using the $24,800 top of the 2026 MFJ 10% bracket, tax is exactly $2,480 + 12% × $20,164.29 = $4,899.71, which leaves $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It itemized $40,772 by treating $24,980 as medical expenses, which counts the duplicated $12,000 premium fields twice and includes non-deductible OTC costs. Correct medical deductions are about $5,430, so itemized deductions fall below the $32,200 standard deduction. Taxable income is $44,964 and tax of $4,899.71 exceeds the $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It never applied the $14,839 qualified overtime deduction and itemized $38,959 by double-counting premiums and including OTC costs. It also claimed a $1,000 AOTC with zero qualified tuition expenses and used $2,000 per child for the CTC instead of $2,200." +us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left out the $14,839 overtime deduction, used a $31,500 standard deduction, and claimed a full $2,500 AOTC even though there were no qualified expenses. It then submitted $8,838, which contradicts its own stated $342 result. The correct calculation gives $4,899.71 of tax less the $4,400 CTC." us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It treated the refundable CTC as the $1,700-per-child cap ($3,400) and never applied the rule that only CTC left after tax is refundable. Tax before credits (about $4,900) uses up the whole CTC, so refundable CTC is $0. Its submitted $3,200 does not even match its own $3,400 figure." us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly zeroed the EITC and found the CTC fully used against tax. It then assumed the head had enough tuition to max out the AOTC and claimed its $1,000 refundable part. No qualified tuition and related expenses are listed, so the AOTC and its refundable 40% are both $0." us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It counted the $4,400 CTC as refundable along with an assumed $1,000 AOTC and reached $4,966 of credits offsetting tax. That mixes nonrefundable credits with refundable ones. The CTC is fully used against tax of about $4,900, leaving $0 refundable CTC. The AOTC is $0 because no qualified tuition expenses are listed." @@ -4146,52 +4189,53 @@ us,scenario_046,payroll_tax,minimax-m3,llm_error,other,False,"The model said it us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly reached $5,704 + $1,334 = $7,038, but it submitted 8438. The final value was mistyped and does not match its own calculation." us,scenario_046,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the $75,303 taxable income right but used the pre-2026 schedule (0.25%–4.75%, top rate above $12,200) instead of the 2026 schedule (0% to $7,500, then 2.5%/3.5%, then 4.5% above $14,400). It also based the credit on a $4,000 federal CTC instead of $4,400, giving $200 instead of $220. Its submitted $3,364 doesn't even match its own $3,022 result." -us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The taxable income ($75,303) and the $220 child credit were both right. The model then used the retired 0.25%–4.75% brackets, which give $3,221.89, instead of the 2026 HB 2764 schedule (0%/2.5%/3.5%/4.5%), which gives $2,959.14 before the credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from federal taxable income with a $14,600 standard deduction instead of federal AGI less Oklahoma's $12,700 standard deduction and $4,000 of exemptions. It applied a flat 5.85% rate that Oklahoma doesn't use, then subtracted an arbitrary amount to reach $1,614 instead of using the 2026 brackets and the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"The model never computed Oklahoma tax and entered $0 as a placeholder. On $75,303 of taxable income, Oklahoma's 2026 joint brackets produce $2,959.14, and the $220 child credit brings that to $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model computed $75,303 of taxable income correctly but applied the old 0.25%–4.75% brackets (about $3,221) instead of the 2026 schedule with a 0% first bracket and a 4.5% top rate ($2,959.14). Its credit was an imprecise figure of about $211 rather than 5% of the $4,400 federal CTC ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model claimed $17,000 of capped itemized deductions. Oklahoma requires the $12,700 standard deduction when the filer doesn't itemize federally, and the $32,200 federal standard deduction is larger than this household's itemized total. It also used the old 4.75% top rate and left out the $220 Oklahoma child credit (5% of the federal CTC)." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached about $75,300 of taxable income but never ran the 2026 brackets. Those brackets (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400, 4.5% above) give $2,959.14, and subtracting the $220 credit leaves $2,739.14. Its rounded $2,500 understates that by about $239." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model assumed Oklahoma conforms to the federal standard deduction and subtracted $30,000. Oklahoma's joint standard deduction is fixed at $12,700, so taxable income is $75,303, not $58,003. It also used a made-up bracket schedule with a 4.75% top rate and left out the $220 child credit (5% of the federal CTC)." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model itemized $20,879 on the Oklahoma return. Oklahoma allows itemizing only when the filer itemizes federally, and here the federal standard deduction is larger, so the $12,700 Oklahoma standard deduction applies and taxable income is $75,303. It also used the old 4.75% top rate and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used itemized deductions and got about $49k of taxable income. Because the household takes the standard deduction federally, Oklahoma requires its $12,700 standard deduction, so taxable income is $75,303. It also used the old 4.75% top rate instead of the 2026 4.5% schedule." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model set the Oklahoma standard deduction equal to the federal one (about $30,900), but Oklahoma's joint standard deduction is fixed at $12,700, and it also left out the $4,000 of exemptions. It used the old 4.75% brackets and figured the credit on a $4,000 federal CTC ($200) instead of $4,400 ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used federal itemized deductions to reach $51,795.62 of taxable income. The household takes the standard deduction federally, so Oklahoma's $12,700 standard deduction applies, and with $4,000 of exemptions taxable income is $75,303. It then used the old 4.75% brackets and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model subtracted $26,958.78 of Oklahoma itemized deductions instead of the $12,700 standard deduction, which Oklahoma requires because the household doesn't itemize federally. It applied the retired 0.25%–4.75% brackets and said no nonrefundable credit applied, leaving out the 5% of the $4,400 federal CTC ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model cut Oklahoma AGI to $77,164 by subtracting the $14,839 overtime premium. The federal overtime deduction is taken below the line and doesn't reduce the $92,003 federal AGI that Oklahoma starts from. It also used a 4.75% top rate instead of the 2026 4.5% schedule and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model claimed $17,000 of capped itemized deductions plus $5,430 of medical expenses instead of the $12,700 Oklahoma standard deduction, which applies because the household takes the federal standard deduction. It used the old 4.75% brackets and figured the credit as 5% of $2,000 ($100) instead of 5% of the $4,400 federal CTC ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"There is no reasoning to check, but the correct figure is $2,959.14 in 2026 bracket tax on $75,303, less the $220 child credit. The submitted $3,154 is close to what the retired 0.25%–4.75% schedule gives ($3,221.89) after a partial credit, so the 2026 0%/2.5%/3.5%/4.5% schedule wasn't used." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model said it used itemized deductions, and its $2,105 matches about $51,796 of taxable income under the old 4.75% brackets. Oklahoma requires the $12,700 standard deduction for a filer who doesn't itemize federally, which gives $75,303 of taxable income. The model also left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $9,208 of ESI premiums to get $82,795 of Oklahoma AGI, but federal AGI, which Oklahoma starts from, is $92,003. It also itemized $26,999 instead of taking the $12,700 standard deduction. It used the old 4.75% brackets and took a $100 credit instead of 5% of the $4,400 federal CTC ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"There is no reasoning to check, but the correct figure is $2,959.14 in 2026 bracket tax on $75,303, less the $220 child credit. The submitted $3,043 is close to the $3,064.14 that comes from keeping the old 0.25%–3.75% lower brackets with a 4.5% top rate, which means the model missed the 0% bracket on the first $7,500 and didn't take the full $220 credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied itemized deductions on the Oklahoma return. Because the household takes the larger federal standard deduction, Oklahoma requires its $12,700 standard deduction, and with $4,000 of exemptions taxable income is $75,303. That base yields $2,959.14 before the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the standard deduction, the exemptions and the child credit, but its bracket tax is wrong. On $75,303 the 2026 joint schedule gives $218.50 for the bands up to $14,400 plus 4.5% of $60,903, for $2,959.14, and subtracting $220 leaves $2,739.14. The model's $2,684.51 comes from mis-specified brackets." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $12,200 standard deduction instead of $12,700, giving $75,803 of taxable income instead of $75,303. It applied a 4.75% top rate instead of the 2026 4.5% schedule and figured the credit on a $4,000 federal CTC ($200) instead of $4,400 ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value for state_income_tax_before_refundable_credits, so there was no answer to score against $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model subtracted the federal overtime deduction from AGI to get $77,164. It also applied the $32,200 federal standard deduction instead of Oklahoma's fixed $12,700, skipped the $4,000 of exemptions, used a flat 4.25% rate instead of the graduated 2026 schedule, and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model never computed Oklahoma tax and entered $0 as a placeholder. All the inputs were given: $92,003 of AGI less the $12,700 standard deduction and $4,000 of exemptions gives $75,303, which is $2,959.14 in 2026 bracket tax, or $2,739.14 after the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model used itemized deductions instead of the required $12,700 Oklahoma standard deduction. It also counted the federal American Opportunity Credit as a nonrefundable Oklahoma credit, but Oklahoma has no such credit; the only nonrefundable credit here is 5% of the federal CTC, or $220." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model got the $75,303 taxable income and the $220 credit right, but it computed $3,064 by keeping the old 0.25%–3.75% lower brackets up to $12,200 with a 4.5% top rate. The 2026 schedule taxes the first $7,500 at 0%, the next $2,300 at 2.5% and $9,800–$14,400 at 3.5%, which gives $2,959.14." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The submitted $4,223.29 is more than Oklahoma rates would charge on the entire $92,003 of AGI, so the model didn't apply the $12,700 standard deduction and $4,000 of exemptions to the 2026 schedule. The federal overtime deduction it mentioned doesn't affect Oklahoma income either. The correct computation is $75,303 of taxable income, $2,959.14 of tax, less the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model claimed $17,000 of capped itemized deductions. Oklahoma requires the $12,700 standard deduction because the household takes the larger federal standard deduction, so taxable income is $75,303, not $71,003. Its $3,018 of bracket tax also comes from the old 4.75% schedule rather than the 2026 4.5% schedule." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model applied a flat 4.5% rate to a base of about $43,303 instead of running the graduated 2026 schedule on $75,303, which means it subtracted a much larger standard deduction than Oklahoma's $12,700. It also used $100 per child ($200 total) instead of 5% of the $4,400 federal CTC ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model got the taxable income and the $220 credit right but computed $2,981.14 of bracket tax instead of $2,959.14. The bands up to $14,400 come to $218.50 (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400), and the model's figure is $22 too high, so it mis-set the lower-bracket thresholds or rates." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $2,513 is $226 below the correct $2,739.14. That gap fits taxing only the 4.5% top band ($2,740.64) and dropping the $218.50 owed in the 2.5% and 3.5% brackets before subtracting the credit." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the correct $220 credit to $3,064.14 of bracket tax, which comes from keeping the old 0.25%–3.75% lower brackets up to $12,200 with a 4.5% top rate. The 2026 HB 2764 schedule has a 0% first bracket to $7,500, 2.5% and 3.5% bands to $14,400 and 4.5% above, which gives $2,959.14." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model applied 4.5% above $14,400 ($2,740.64) correctly, but it computed only $66.50 on the first $14,400. The correct amount is $218.50: 2.5% on $7,500–$9,800 plus 3.5% on $9,800–$14,400. That understated the pre-credit tax by $152." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's $3,800 is higher than the retired 4.75% schedule gives on the correct $75,303 ($3,221.89), so it overstated both the base and the rates. It also left out the $220 Oklahoma child credit, which is 5% of the federal CTC." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted $26,309 of itemized deductions instead of the $12,700 Oklahoma standard deduction, which applies because the household takes the larger federal standard deduction. It then used the old 4.75% brackets and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model itemized $26,309 on the Oklahoma return. Oklahoma requires the $12,700 standard deduction when the filer doesn't itemize federally, so taxable income is $75,303, not $61,694. It also used the old 4.75% rate above $12,200 and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted $26,309 of itemized deductions instead of the $12,700 standard deduction, which Oklahoma requires because the household takes the federal standard deduction. It used the old 4.75% brackets and said no Oklahoma nonrefundable credits applied, leaving out the $220 credit (5% of the $4,400 federal CTC)." -us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied a flat 4.5% rate to $45,216 of income after itemized deductions. Oklahoma taxable income is $75,303, after the $12,700 standard deduction and $4,000 of exemptions, and it is taxed on the graduated 2026 schedule (0%/2.5%/3.5%/4.5%), less the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model started from $77,164 of AGI by subtracting the federal overtime deduction, which is taken below the line and leaves AGI at $92,003. It used a standard deduction of about $15,000 instead of Oklahoma's $12,700 and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value for state_income_tax_before_refundable_credits, so there was no answer to score against $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model left out the four $1,000 Oklahoma exemptions and got $79,303 of taxable income instead of $75,303. It used the retired 4.75% brackets and said no nonrefundable credit applied, missing the Oklahoma child credit of 5% of the $4,400 federal CTC ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"There is no reasoning to check, but the model's $3,773 is above even what the retired 4.75% schedule gives on the correct $75,303 base ($3,221.89). The correct computation applies the 2026 0%/2.5%/3.5%/4.5% brackets for $2,959.14 and subtracts the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied the 4.5% top rate but kept the old 0.25%–3.75% lower brackets up to $12,200, which gives $3,064.14 instead of $2,959.14 under the 2026 schedule. It also checked only the Oklahoma EITC and child care credit and missed that the Oklahoma child care/child tax credit is the greater of 20% of the federal CDCC or 5% of the federal CTC, which here is $220." -us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model misadded AGI as $94,004 when it is $92,003 and left out the $4,000 of exemptions, which produced $81,304 of taxable income instead of $75,303. It used the old brackets and left out the $220 child credit." -us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed federal taxable income ($60,503). Oklahoma starts from federal AGI and subtracts its own $12,700 standard deduction and $4,000 of exemptions, which gives $75,303. It also made up a 0.5%–5% bracket schedule and left out the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It found the correct $75,303 taxable income but applied the pre-2026 0.25%–4.75% bracket schedule, getting $3,222 instead of $2,959.14 under the 2026 0%/2.5%/3.5%/4.5% schedule. It used a $200 credit based on a $2,000-per-child CTC instead of $220 (5% of $4,400). It then submitted $3,364, which contradicts its own $3,022 result." +us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Taxable income ($75,303) and the $220 credit are correct, but it taxed the income under the pre-2026 0.25%–4.75% brackets ($3,221.89). The 2026 MFJ schedule is 0% to $7,500, 2.5% to $9,800, 3.5% to $14,400 and 4.5% above, which gives $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It started from federal taxable income using a $14,600 federal standard deduction and applied a nonexistent flat 5.85% rate, skipping the $12,700 Oklahoma standard deduction and the four $1,000 exemptions. It then cut that result by arbitrary credits to reach $1,614, instead of $2,959.14 tax on $75,303 minus the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed Oklahoma tax and submitted a $0 placeholder. Taxable income is $75,303 ($92,003 − $12,700 − $4,000), which produces $2,959.14 under the 2026 brackets, less the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the correct $75,303 taxable income but applied the pre-2026 brackets with a 4.75% top rate above $12,200 (about $3,221), instead of the 2026 schedule that taxes $60,903 above $14,400 at 4.5% for $2,959.14. It also guessed the credit at about $211 instead of exactly 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return using the $17,000 cap. Because the household takes the federal standard deduction, Oklahoma requires the $12,700 standard deduction, so taxable income is $75,303, not $71,003. It also used the old 4.75% bracket schedule and omitted the $220 nonrefundable child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income (~$75,300) is right, but its rough $2,500 figure does not come from the 2026 schedule. That schedule gives $57.50 + $161 + 4.5% × $60,903 = $2,959.14, and subtracting the $220 credit leaves $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed Oklahoma adopted the federal standard deduction and subtracted $30,000. Oklahoma's MFJ standard deduction is a fixed $12,700, so taxable income is $75,303, not $58,003. It also used pre-2026 brackets with a 4.75% top rate and left out the $220 credit (5% of the federal CTC)." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It first claimed Oklahoma has no income tax, then itemized $20,879 on the Oklahoma return. A household that takes the federal standard deduction must use Oklahoma's $12,700 standard deduction. It also applied a 4.75% top rate and skipped the $220 child credit, giving $3,200 instead of $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used inflated itemized deductions to reach about $49k of taxable income. Because the household takes the federal standard deduction, Oklahoma requires the $12,700 standard deduction, so taxable income is $75,303 and tax is $2,959.14 before the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It wrongly set the Oklahoma standard deduction equal to the federal MFJ amount (~$30,900) instead of $12,700 and omitted the four $1,000 exemptions. It also used old 4.75% brackets and a $200 credit based on a $2,000-per-child CTC instead of $220." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It carried federal itemized deductions into Oklahoma, getting $51,795.62 of taxable income. A household that takes the federal standard deduction uses Oklahoma's $12,700 standard deduction, which leaves $75,303. It also used the old 0.25%-start brackets instead of the 2026 0%/2.5%/3.5%/4.5% schedule and dropped the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted $26,958.78 of Oklahoma itemized deductions. Because the household takes the federal standard deduction, the $12,700 Oklahoma standard deduction applies. It also stated that no nonrefundable state credits apply, missing the $220 credit equal to 5% of the federal CTC." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It reduced Oklahoma AGI to $77,164 by subtracting the federal overtime deduction. That deduction is taken after AGI and does not lower the $92,003 federal AGI that Oklahoma starts from. It also used pre-2026 brackets and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return ($17,000 cap plus $5,430 medical) even though the household takes the federal standard deduction, which forces the $12,700 Oklahoma standard deduction. It also computed the child credit as $100 using a single $2,000 CTC instead of 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation. Its $3,154 matches taxing $75,303 under the pre-2026 4.75% schedule with little or no credit, rather than $2,959.14 under the 2026 0%/2.5%/3.5%/4.5% brackets less the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted itemized deductions on the Oklahoma return. Because the household takes the federal standard deduction, Oklahoma requires the $12,700 standard deduction, so taxable income is $75,303 and the tax after the $220 credit is $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It lowered Oklahoma AGI to $82,795 instead of the $92,003 federal AGI and subtracted $26,999 of Oklahoma itemized deductions instead of the required $12,700 standard deduction. It also used a $100 child credit instead of 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. Its $3,043 is about $304 too high, which matches taxing $75,303 under the pre-2026 4.75% brackets instead of the 2026 schedule ($2,959.14) and then subtracting the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used itemized deductions on the Oklahoma return. A household that takes the federal standard deduction must use Oklahoma's $12,700 standard deduction, which gives $75,303 of taxable income and $2,739.14 after the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It named the right steps (standard deduction, exemptions, child credit) but miscomputed the bracket tax on $75,303. The 2026 schedule gives $2,959.14, and $2,959.14 − $220 = $2,739.14; its $2,684.51 is $54.63 short." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $12,200 standard deduction instead of $12,700 and applied an old 4.75% top-rate schedule ($3,178.64) instead of the 2026 0%/2.5%/3.5%/4.5% brackets. It also took 5% of a $4,000 CTC ($200) instead of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_046,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted the federal overtime deduction from Oklahoma AGI ($77,164) and used the $32,200 federal standard deduction instead of Oklahoma's $12,700, with no $1,000 exemptions. It then applied a flat 4.25% rate instead of the graduated 2026 brackets and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"It submitted a $0 placeholder instead of computing the tax. The facts fully determine $75,303 of Oklahoma taxable income, $2,959.14 of bracket tax, and a $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It used itemized deductions on the Oklahoma return, although Oklahoma requires the $12,700 standard deduction when the household takes the federal standard deduction. It also applied the federal American Opportunity Credit against state tax, while the only state nonrefundable credit is the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its taxable income ($75,303) and $220 credit are correct, but its $3,064 regular tax applies 4.5% above $12,200 on top of the old 0.25%–3.75% lower brackets. The 2026 restructuring sets 0% up to $7,500, 2.5% to $9,800 and 3.5% to $14,400, with 4.5% only above $14,400, which gives $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It subtracted the federal overtime deduction from Oklahoma income even though Oklahoma starts from the $92,003 federal AGI. Its $4,223.29 is higher than any bracket tax possible on $75,303, so the bracket arithmetic is also wrong. The correct result is $2,959.14 less the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied a $17,000 capped itemized deduction. Because the household takes the federal standard deduction, Oklahoma requires the $12,700 standard deduction, so taxable income is $75,303, not $71,003, and the tax is $2,959.14 before the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied a flat 4.5% rate instead of the graduated 2026 brackets. Its taxable income (~$43,303) implies deductions far above the $12,700 Oklahoma standard deduction. It also used two $100 credits instead of 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Taxable income ($75,303) and the $220 credit are correct, but its bracket tax of $2,981.14 counts $240.50 for income below $14,400. The 2026 schedule yields $218.50 on that slice (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400), so the total is $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Its $2,513 understates the tax by $226. Oklahoma taxable income of $75,303 under the 2026 MFJ brackets gives $2,959.14, and subtracting the $220 child credit leaves $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its deduction, exemptions and $220 credit are correct, but its pre-credit tax of $3,064.14 applies 4.5% above $12,200 on top of the old lower brackets. The 2026 schedule starts the 4.5% rate above $14,400 and taxes the first $7,500 at 0%, which gives $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Taxable income ($75,303) and the $220 credit are correct, but its bracket tax of $2,807.14 is $152 below the $2,959.14 that the 2026 schedule produces ($57.50 + $161 + 4.5% × $60,903)." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no computation, and its $3,800 is more than even the old 4.75% schedule produces on $75,303. The correct path is $2,959.14 under the 2026 brackets less the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted $26,309 of Oklahoma itemized deductions, although a household that takes the federal standard deduction must use Oklahoma's $12,700 standard deduction. It also omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized $26,309 on the Oklahoma return instead of taking the required $12,700 standard deduction, and it used the old 4.75%-above-$12,200 schedule instead of the 2026 4.5%-above-$14,400 brackets. It also left out the $220 credit (5% of the federal CTC)." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used $26,309 of itemized deductions on the Oklahoma return. Because the household takes the federal standard deduction, the $12,700 standard deduction applies. It also wrongly said no nonrefundable Oklahoma credits apply, missing the $220 credit equal to 5% of the federal CTC." +us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied a flat 4.5% rate to $45,216 of itemized-based taxable income, ignoring the graduated 2026 brackets and the required $12,700 standard deduction that gives $75,303 of taxable income. It also omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $77,164 by subtracting the federal overtime deduction, which does not reduce the $92,003 federal AGI. It also used a ~$15,000 standard deduction instead of Oklahoma's $12,700 and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It left out the four $1,000 Oklahoma personal exemptions (taxable income $79,303 instead of $75,303) and used the old 4.75% bracket schedule. It also wrongly concluded no Oklahoma credit applies because the federal CDCC is zero, but the credit can be 5% of the federal CTC, which here is $220." +us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It gave no computation, and its $3,773 is more than any bracket schedule produces on the $75,303 taxable income. The correct result is $2,959.14 under the 2026 brackets less the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used a $13,550 standard deduction instead of Oklahoma's fixed $12,700. Its $3,182 graduated tax reflects the old 4.75% schedule rather than the 2026 0%/2.5%/3.5%/4.5% brackets, which give $2,959.14 on $75,303." +us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It kept the old lower brackets (0.25%–3.75% up to $12,200) and applied 4.5% only above that, getting $3,064.14 instead of $2,959.14 under the 2026 schedule. That schedule taxes the first $7,500 at 0% and applies 4.5% above $14,400. It also omitted the $220 credit, wrongly treating the Oklahoma child credit as zero because the federal CDCC is zero." +us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It overstated AGI as $94,004 instead of $92,003 and omitted the four $1,000 exemptions, giving $81,304 of taxable income instead of $75,303. It also used pre-2026 brackets and left out the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It taxed federal taxable income ($60,503) with an invented 0.5%–5% schedule. Oklahoma's base is federal AGI minus the $12,700 standard deduction and $4,000 of exemptions ($75,303), taxed under the 2026 0%/2.5%/3.5%/4.5% brackets. It also ignored the $220 nonrefundable child credit." us,scenario_046,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_048,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age and assets below $2,000 as sufficient for Ohio aged Medicaid while never establishing that the head passed the applicable income test. The head's wages and Social Security total $34,626, SSI is zero, and no Medicaid category applies." @@ -4210,49 +4254,50 @@ us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals us,scenario_048,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and assigned an estimated $225 monthly benefit without applying the SNAP income threshold. That income exceeds the $17,420 limit for a one-person household, producing $0." us,scenario_048,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate to the first dollar of taxable income, ignoring Ohio's zero-tax bracket: Ohio's rate schedule imposes no tax on Ohio taxable nonbusiness income at or below the roughly $26,050 threshold, and this filer's $7,656 of wages (Social Security survivor benefits are excluded from federal AGI here because provisional income of $21,141 is under the $25,000 base amount) falls far below it, yielding $0. It compounded the error by inventing a nonexistent $1,400 Ohio 'single standard deduction' — Ohio starts from federal AGI and allows personal exemptions, not a standard deduction — and then taxed the residual $3,856 at 2.75% to reach $106." us,scenario_048,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model taxed the entire $7,656 of Ohio taxable income at 2.75% ($210.54), skipping Ohio's zero-rate first bracket that exempts all Ohio taxable nonbusiness income up to about $26,050, under which this household's liability is $0 before any credit is considered. It also mis-specified the personal exemption as a credit equal to 2.75% of $2,400 ($66) — Ohio subtracts the $2,400 exemption from Ohio adjusted gross income and separately allows only a $20-per-exemption credit — and then netted a $50 senior citizen credit against a tax that never existed, landing on $94.54." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own derivation reached $30,830. That figure omits the 2026 non-itemizer cash charitable deduction (~$580, worth about $128 of tax) but is otherwise correct. It then submitted $33,223, a number unrelated to its arithmetic." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction and the non-itemizer charitable deduction. It also deducted the $721 traditional IRA, which is fully phased out for an active plan participant at this income, and used a stale $29,200 standard deduction instead of $32,200. The $48,837 it reported is not supported by any bracket calculation; correct 2026 brackets on its own $217k taxable income give about $36.6k." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It reported $0 tax on $247,385 of AGI. The correct path is $189,605 of taxable income after the $32,200 standard deduction, $25,000 overtime deduction and ~$580 charitable deduction, which yields $30,703 of tax with no nonrefundable credits to offset it." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction, and used a $31,500 standard deduction instead of $32,200. It also wrongly added the 0.9% Additional Medicare Tax to income tax. It then submitted $49,861, which does not match its own $36,571 calculation." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction, and used a $30,300 standard deduction with outdated bracket thresholds. Even so, its own work produced $37,055, yet it submitted an unrelated $53,503." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the $40,000 SALT cap as the deductible amount, but only the $20,655 of real estate taxes actually paid is deductible. That inflated itemized deductions to $46,081 when they are under $27k, below the $32,200 standard deduction. It also missed the $25,000 qualified overtime deduction, which is available whether or not the household itemizes." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction ($25,000 joint cap, no phase-out at $247k MAGI) and the non-itemizer cash charitable deduction. It also used the 2025 standard deduction of $30,000 and 2025 bracket thresholds instead of 2026's $32,200 and $24,800/$100,800/$211,400." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 overtime deduction and the charitable deduction, and deducted the $721 IRA contribution even though that deduction is fully phased out. Its own estimate was about $38,831, but it then inflated the answer to $62,700 by adding Medicare and phase-out effects that do not belong in income tax." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $928 of auto-loan interest. That deduction drops by $200 per $1,000 of MAGI over $200,000, so at $247,385 it is fully eliminated. It also missed the ~$580 non-itemizer cash charitable deduction; the net $348 of extra deductions explains the $77 shortfall." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction or the ~$580 non-itemizer charitable deduction, and used a $31,000 standard deduction instead of $32,200. That overstated taxable income by more than $26k." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA provisions expired in 2026 and used pre-TCJA 15/25/28% brackets, personal exemptions and uncapped itemizing. OBBBA made the TCJA rates and the $32,200 standard deduction permanent and added the $25,000 overtime deduction, which it omitted. It also subtracted the $16,408 employer-sponsored insurance premium from wages, although only the traditional 401(k) reduces the stated wages." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted the employer-sponsored insurance premium from wages and did not apply the $25,000 qualified overtime deduction or the non-itemizer charitable deduction. Its $36,639 implies taxable income of roughly $215k against the correct $189,605." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI, standard deduction and 2026 brackets were correct, but it omitted the $25,000 qualified overtime deduction and the $579.76 non-itemizer charitable deduction. That gap is exactly 22% × $25,579.76 = $5,627.55, the full amount of its overstatement." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA provisions expired and used pre-TCJA brackets, personal exemptions and uncapped itemized deductions. Under OBBBA the 2026 rules are the $32,200 standard deduction, TCJA-style brackets and the $25,000 overtime deduction, which it omitted. It also subtracted the $16,408 employer-sponsored insurance premium from the stated wages." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used about $229,000 of taxable income, which means it skipped the $25,000 qualified overtime deduction and the full $32,200 standard deduction. Correct taxable income is $189,605." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-expired law (personal exemptions, uncapped real estate taxes, pre-TCJA brackets), but OBBBA kept the TCJA structure for 2026. It omitted the $25,000 qualified overtime deduction and wrongly subtracted the employer-sponsored insurance premium from wages." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used TCJA-expired rules (personal exemptions of $10,100, 25% bracket, itemizing $26,736) instead of 2026 OBBBA law, which has a $32,200 standard deduction and TCJA brackets. It never applied the $25,000 qualified overtime deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $42,203 implies taxable income far above the correct $189,605, consistent with skipping the $25,000 qualified overtime deduction and the non-itemizer charitable deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from the stated gross wages, but only the $15,436 traditional 401(k) reduces them. It also used a $30,000 standard deduction and omitted the $25,000 qualified overtime deduction and the ~$580 charitable deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reported $37,921 with no overtime deduction mentioned. That amount reflects taxable income above $215k, so it left out the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction that bring taxable income to $189,605." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets instead of 2026's $32,200 and the $100,800 top of the 12% bracket. It also omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer cash charitable deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there was nothing to score." -us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the correct $32,200 standard deduction but never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction. It also miscomputed ordinary tax: $36,982 instead of $35,399 on $208,977 under 2026 brackets." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It produced $20,460 with no computation. That understates the tax on $189,605 of taxable income, which is $29,771 of ordinary tax plus $931 on qualified dividends." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $38,566 reflects taxable income about $35k above the correct $189,605, consistent with omitting the $25,000 qualified overtime deduction and using a smaller standard deduction or older brackets." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from wages, which cut AGI to $230,977. Only the $15,436 traditional 401(k) reduces the stated gross wages, so AGI is $247,385. The extra exclusion understated taxable income by $16,408, or $3,610 of tax at 22%." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It took the $928 qualified vehicle-loan interest deduction. That deduction drops by $200 per $1,000 of MAGI over $200,000, so at $247,385 it is zero. The extra $928 of deductions accounts for the $204 shortfall." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 employer-sponsored insurance premium as a pre-tax reduction of the stated gross wages. Only the $15,436 traditional 401(k) is excluded, so AGI is $247,385, not $230,977. That understated taxable income by $16,408." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $33,473 implies taxable income of about $202,200, so it allowed only about $13,000 of overtime and vehicle-loan deductions. The correct deductions are the full $25,000 overtime deduction plus the ~$580 non-itemizer charitable deduction, and the vehicle-loan deduction is fully phased out above $200k MAGI." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from wages, giving AGI of $230,977 instead of $247,385. Only the traditional 401(k) reduces the stated gross wages, and the extra exclusion understated tax by about $3,610." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the 2026 brackets, the standard deduction and the $25,000 overtime deduction correctly, but omitted the 2026 non-itemizer cash charitable deduction ($579.76). That accounts exactly for the $127.55 overstatement (22% × $579.76)." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It excluded the $16,408 employer-sponsored insurance premium from wages, although only the $15,436 traditional 401(k) reduces the stated gross wages. That understated AGI and taxable income by $16,408 and tax by about $3,610." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It used AGI of about $263k (no 401(k) exclusion), took about $20k of itemized deductions instead of the $32,200 standard deduction, and omitted the $25,000 overtime deduction. That produced taxable income of about $230k against the correct $189,605." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA-expired 2026 law: personal exemptions of $10,500, uncapped property taxes and pre-TCJA brackets. Under OBBBA the TCJA brackets and the $32,200 standard deduction continue, and the $25,000 qualified overtime deduction, which it omitted, is added." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-expired rules: 10/15/25% brackets, about $10,910 of personal exemptions and itemizing $26,736. 2026 OBBBA law uses the $32,200 standard deduction and TCJA brackets, plus the $25,000 qualified overtime deduction, which it never took." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the ~$580 non-itemizer cash charitable deduction, so taxable income came out at $190,185 instead of $189,605. It also miscomputed ordinary tax on $183,977 as $30,051; the 2026 brackets give $29,899." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It itemized with a capped SALT deduction and never applied the $25,000 qualified overtime deduction or the $32,200 standard deduction. The $36,370 it reported implies taxable income of about $214k against the correct $189,605." -us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its AGI of $230,256 subtracts both the $16,408 employer-sponsored insurance premium and the phased-out $721 IRA contribution, neither of which reduces AGI. It also omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used TCJA-expired rules (personal exemptions, pre-TCJA brackets, itemizing) and subtracted the employer-sponsored insurance premium from wages. It omitted the $25,000 qualified overtime deduction and the $32,200 standard deduction that apply under 2026 OBBBA law." -us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It ran its full $190,089 taxable income, including the $6,208 of qualified dividends, through the ordinary brackets and then taxed those dividends again at 15%, counting them twice. It also used a made-up $25,675 overtime cap (the correct deduction is $25,000), deducted the phased-out $721 IRA, and used a $30,900 standard deduction instead of $32,200." -us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified overtime deduction and the ~$580 non-itemizer charitable deduction. It also deducted the $721 traditional IRA, which is fully phased out for an active plan participant at $247k MAGI." -us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It made three deduction errors. It miscomputed AGI as $258,821 ($262,821 − $15,436 = $247,385), chose $15,395 of itemized deductions over the larger $32,200 standard deduction, and omitted the $25,000 overtime deduction. It then submitted $51,215, which does not match its own $42,933 figure." -us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It deducted the phased-out $721 IRA plus Roth contributions ($18,160 total), when only the $15,436 traditional 401(k) is excludable. It used a $32,900 standard deduction and omitted the $25,000 overtime deduction. It also overcomputed ordinary tax: $44,587 on $205,441, when 2026 brackets give about $34.4k." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its own work reached $30,830 using the correct AGI, the $32,200 standard deduction and the $25,000 overtime deduction. It left out the 2026 non-itemizer cash charitable deduction ($579.76), which accounts for the $127.55 gap. It then submitted $33,223, a number with no connection to its own calculation." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the old $29,200 standard deduction and deducted the $721 IRA contribution, which is phased out. It never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction. Its $48,837 is also far above what the 2026 joint brackets give even for its own $217,364 of taxable income." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It reported $0 tax on $247,385 of AGI with no calculation. The 2026 joint brackets applied to $189,605 of taxable income give $30,702.59." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It skipped the $25,000 OBBBA overtime deduction and the $579.76 non-itemizer charitable deduction, and used a $31,500 standard deduction instead of $32,200. It also added Additional Medicare Tax, which is a payroll tax and not part of income tax. It then submitted $49,861, a number unrelated to its own $36,571 calculation." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction, and it used a $30,300 standard deduction with outdated brackets instead of $32,200 and the 2026 brackets. It then submitted $53,503, which does not match its own $37,055 calculation." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the roughly $40,000 SALT cap itself as the deduction instead of the actual $20,655 of real estate taxes. That made itemized deductions look like $46,081 when they are about $25k, below the $32,200 standard deduction. It also omitted the $25,000 overtime deduction, and its $47,500 does not match even its own taxable income." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $30,000 standard deduction and 2025-era brackets instead of the 2026 values ($32,200, 22% bracket starting at $100,800). It never applied the OBBBA qualified overtime deduction ($25,000 cap on the $44,647 overtime premium) or the non-itemizer charitable deduction, so taxable income came out about $27.8k too high." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted the $721 IRA contribution, which is phased out, and omitted both the $25,000 overtime deduction and the charitable deduction. It then inflated its own $38,831 estimate to $62,700 by citing Additional Medicare wages, which are not part of income tax." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $928 auto-loan interest even though the OBBBA car-loan deduction drops by $200 for every $1,000 of MAGI over $200k on a joint return, which wipes it out at $247,385. It also skipped the $579.76 non-itemizer charitable deduction. The net $348 understatement of taxable income explains the $76.59 shortfall." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified overtime deduction and the non-itemizer charitable deduction. It also used a $31,000 standard deduction and off-schedule brackets instead of $32,200 and the 2026 joint brackets." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA had sunset and used pre-TCJA brackets with personal exemptions, although OBBBA made the TCJA structure permanent with a $32,200 standard deduction for 2026. It also subtracted the $16,408 ESI premium from wages and omitted the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premium from wages, but the only pre-tax payroll reduction is the $15,436 traditional 401(k). It also left out the $25,000 qualified overtime deduction, which pushed tax about $6k over the reference." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It got AGI, the $32,200 standard deduction and the brackets right, but skipped the $25,000 qualified overtime deduction and the $579.76 non-itemizer charitable deduction. Its $36,330.14 is exactly the reference plus 22% of $25,579.76." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (personal exemptions, 15%/25% brackets) instead of the OBBBA-permanent 2026 schedule with the $32,200 standard deduction. It also excluded the $16,408 ESI premium from wages and omitted the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $229,000 leaves out the $25,000 qualified overtime deduction and the charitable deduction; the correct figure is $189,605.23. Its $52,367.59 is also well above what the 2026 joint brackets give even on that figure." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used TCJA-sunset law (uncapped itemized real estate taxes plus $10,800 of personal exemptions) instead of the 2026 OBBBA rules. It also cut AGI to $230,977 by treating the $16,408 ESI premium as a pre-tax exclusion, and never applied the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration rules (itemized deductions plus $10,100 of exemptions, 25% bracket) instead of the 2026 OBBBA schedule. Its $194,141 of taxable income assumes the $16,408 ESI premium was excluded from wages and leaves out the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $42,203 is more than $11k above the correct figure and is consistent with ignoring the $25,000 qualified overtime deduction and applying wrong brackets to roughly $215k of taxable income." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 ESI premium as a wage exclusion and used a $30,000 standard deduction instead of $32,200. It also omitted the $25,000 qualified overtime deduction and the non-itemizer charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It listed only wages, interest, dividends and the dividend rate preference, never the OBBBA qualified overtime deduction ($25,000) or the non-itemizer charitable deduction. Its $37,920.81 matches tax on roughly $215k of taxable income instead of $189,605." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $30,000 standard deduction instead of the 2026 joint amount of $32,200. It never subtracted the $25,000 qualified overtime deduction or the $579.76 non-itemizer charitable deduction, so taxable income came out at $217,385 instead of $189,605.23." +us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no answer and no reasoning for this output, so there was nothing to score." +us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It had the right AGI and the $32,200 standard deduction but omitted the $25,000 qualified overtime deduction and the $579.76 non-itemizer charitable deduction. It also miscomputed ordinary tax on $208,977: the correct figure is $35,399, not $36,982." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It reasoned from an itemized set with a $10,000 SALT cap and reported $20,460 without a bracket calculation. The correct path ($247,385 AGI less $32,200 standard, $25,000 overtime and $579.76 charitable deductions) gives $30,702.59, so its figure understates tax by about $10k." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $38,566 is consistent with taxing roughly $220k of taxable income, which means it ignored the $25,000 qualified overtime deduction and the 2026 $32,200 standard deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied the standard, overtime and charitable deductions correctly, but subtracted the $16,408 employer-sponsored insurance premium from wages. That gave an AGI of $230,977; only the $15,436 traditional 401(k) reduces wages, so AGI is $247,385. The $16,408 understatement of taxable income at 22% accounts for the $3,610 shortfall." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It claimed the $928 qualified vehicle-loan interest deduction, but that deduction falls by $200 per $1,000 of MAGI over $200k on a joint return and is zero at $247,385. Its $30,498.38 is the reference minus 22% of $928." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It excluded the $16,408 ESI premium from wages, giving taxable income of $173,197 instead of $189,605.23. Only the $15,436 traditional 401(k) is a pre-tax wage reduction. Its other deductions were correct." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It included the vehicle-loan interest deduction, which is fully phased out above $200k MAGI on a joint return, and left out the non-itemizer charitable deduction. Its $33,473 implies about $12.6k more taxable income than the correct $189,605.23, so it also understated the $25,000 overtime deduction or misapplied the 2026 joint brackets." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 employer-sponsored insurance premium as a pre-tax exclusion and cut AGI to $230,977. With AGI at $247,385, the same $32,200 standard, $25,000 overtime and charitable deductions give $189,605.23 of taxable income and $30,702.59 of tax." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It got AGI, the standard deduction, the overtime deduction and the brackets right, but omitted the 2026 non-itemizer cash charitable deduction ($579.76). Its $30,830.14 is exactly the reference plus 22% of $579.76." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 ESI premium from wages along with the 401(k), which understated AGI by $16,408. All of its other deductions and rates match the correct calculation." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It estimated taxable income at about $230k from capped itemized deductions and never applied the $32,200 standard deduction or the $25,000 qualified overtime deduction. The correct taxable income is $189,605.23." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (personal exemptions, uncapped property taxes, pre-TCJA brackets), although OBBBA set a $32,200 standard deduction and permanent TCJA brackets for 2026. It also never applied the $25,000 overtime deduction, which left taxable income at $210,149." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-sunset itemizing, personal exemptions and 10/15/25% brackets instead of the 2026 OBBBA schedule with the $32,200 standard deduction. It also omitted the $25,000 qualified overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the $579.76 non-itemizer cash charitable deduction. It also miscomputed ordinary tax on $183,977: the joint brackets give $29,899, not $30,051." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It reasoned from capped itemized deductions and retirement contributions but never applied the $25,000 qualified overtime deduction or the non-itemizer charitable deduction. That left taxable income near $215k instead of $189,605." +us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its $230,256 AGI subtracts both the $16,408 ESI premium and the $721 IRA contribution. Neither reduces AGI: the premium is not a wage exclusion and the IRA deduction is phased out. It also omitted the $25,000 overtime deduction and the charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA had expired and used itemized deductions plus personal exemptions with pre-TCJA brackets instead of the 2026 OBBBA $32,200 standard deduction. It also excluded the $16,408 ESI premium from wages and ignored the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It ran the full $190,089 of taxable income, qualified dividends included, through the ordinary brackets and then added 15% on the dividends again, which double-counts them. It also deducted the phased-out $721 IRA contribution, used a $25,675 overtime figure instead of the $25,000 joint cap and a $30,900 standard deduction instead of $32,200, and omitted the charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It cut AGI to $242,585 by excluding $4,800 of health premiums from wages, but the only pre-tax wage reduction is the $15,436 traditional 401(k), so AGI is $247,385. It also omitted the $579.76 non-itemizer charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It deducted the $721 traditional IRA contribution, which is fully phased out for an active plan participant at this MAGI. It also never applied the $25,000 qualified overtime deduction or the $579.76 non-itemizer charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It miscomputed AGI as $258,821 ($262,821 − $15,436 is $247,385). It claimed $15,395 of itemized deductions 'exceeds' a $32,000 standard deduction and itemized anyway, and it omitted the $25,000 overtime deduction. It then submitted $51,214.75, which does not match its own $42,933 calculation." +us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $18,160 of 'IRA and 401(k)' deductions includes the after-tax $2,724 Roth 401(k) contribution, and it used a $32,900 standard deduction. It omitted the $25,000 qualified overtime deduction and overstated bracket tax: $44,587 on $205,441 of ordinary income, when the 2026 joint brackets give about $34.7k." us,scenario_049,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -4261,38 +4306,39 @@ us,scenario_049,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_ou us,scenario_049,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's reasoning correctly determined that the 53-year-old head lacks both the age threshold and any qualifying early-Medicare condition, but it submitted value 1 instead of 0. Its numeric output directly contradicts its own eligibility analysis and required final marker." us,scenario_049,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning gets the right total: $11,439 Social Security on the $184,500 base, $3,668.50 Medicare and $27 Additional Medicare Tax, for $15,134.50. It then submitted an unrelated $14,503.98, so the output does not match its own calculation." -us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 wage base of $168,600 instead of the 2026 base of $184,500. It applied the $200,000 single threshold for Additional Medicare Tax instead of the $250,000 joint threshold ($477 instead of $27). It also invented $904.13 of New Hampshire payroll tax, though New Hampshire has none. Its listed parts add to $15,502.83, not the $20,502.83 it submitted." -us,scenario_049,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It said Social Security, Medicare and Additional Medicare Tax apply to the spouse's $253,000 in wages but submitted 0. It left out the $11,439 + $3,668.50 + $27 = $15,134.50 it should have reported." -us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That understates Social Security tax by $520.80 ($10,918.20 instead of $11,439); its Medicare and Additional Medicare Tax figures were correct." -us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning gets the right total: $11,439 + $3,668.50 + $27 = $15,134.50, using the $184,500 base. It then submitted $13,164.85, which does not match its own calculation." -us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It briefly tried an estimated 2026 base of $183,600, then dropped it and used the 2025 base of $176,100. The 2026 base is $184,500, so Social Security tax should be $11,439, not $10,918.20, which left its total $520.80 short." -us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It treated $176,100 (the 2025 base) as the 2026 Social Security wage base. The 2026 base is $184,500, so it understated Social Security tax by $520.80. Its Medicare and Additional Medicare Tax figures were correct." -us,scenario_049,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a made-up 2026 wage base of $180,900 instead of $184,500. That gives $11,215.80 of Social Security tax instead of $11,439; the rest of its figures were correct." -us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,408 employer-sponsored insurance premiums from FICA wages. That cut Medicare tax to $3,430.58 and wrongly removed the $27 Additional Medicare Tax, which applies on all $253,000. It also used the 2025 base of $176,100 instead of $184,500." -us,scenario_049,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Its $14,850.984 equals 1.45% of $236,592 (wages minus the $16,408 premiums) plus 6.2% of $184,200. So it excluded the premiums from Medicare wages, dropping the $27 Additional Medicare Tax, and used a wage base $300 below the 2026 base of $184,500." -us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $16,408 insurance premiums to get $236,592 of Medicare wages. That understated Medicare tax and wrongly removed the $27 Additional Medicare Tax. It also used a guessed wage base of $183,900 instead of $184,500." -us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2025 base of $176,100 instead of the 2026 base of $184,500. Its submitted $13,959.81 is even below the $14,613.70 its own method produces, so it also made an arithmetic error on top of the stale base." -us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut FICA wages to $236,592 by subtracting the $16,408 premiums. That lowered Medicare tax to $3,430.58 and wrongly removed the $27 Additional Medicare Tax. Its $14,814 also implies a $183,600 wage base instead of $184,500." -us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It used $236,592 of Medicare wages, after subtracting the $16,408 premiums, instead of the full $253,000, and so wrongly dropped the $27 Additional Medicare Tax. It also used a guessed 2026 base of $174,900 instead of $184,500." -us,scenario_049,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"Its $4,058.45 is about Medicare tax alone, so it left out nearly all of the $11,439 employee Social Security tax on the spouse's wages, which run past the $184,500 base." -us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums to get $236,592 of FICA wages. That cut Medicare tax and wrongly removed the $27 Additional Medicare Tax on wages above $250,000. It also used a $181,200 base instead of $184,500." -us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $14,613.34 matches 6.2% of the 2025 base of $176,100 plus Medicare and Additional Medicare Tax. It needed the 2026 base of $184,500, which gives $11,439 of Social Security tax." -us,scenario_049,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used $176,100 (the 2025 base) as the Social Security cap instead of the 2026 base of $184,500. That understated Social Security tax by $520.80; its other parts were correct." -us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no payroll_tax value or explanation, so there was no answer to score against $15,134.50." -us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It named the correct parts but submitted $19,648, about $4,500 too high. A figure that high only results if Social Security tax is applied above the $184,500 cap; the capped amount is $11,439." -us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $15,302 is $167.50 too high. That fits a Social Security base above the 2026 cap of $184,500. The correct parts are $11,439 Social Security, $3,668.50 Medicare and $27 Additional Medicare Tax." -us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums from FICA wages. That lowered Medicare wages to $236,592 and set Additional Medicare Tax to $0 instead of $27. It also used a $183,600 base instead of $184,500." -us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"Its Social Security figure ($11,439) is right. It subtracted the $16,408 premiums from Medicare wages, so Medicare tax came out at $3,430.58 instead of $3,668.50, and it skipped the $27 Additional Medicare Tax due on wages above $250,000." -us,scenario_049,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used the correct $184,500 base but subtracted the $16,408 premiums to get $236,592 of Medicare wages. That understated Medicare tax by $237.92 and wrongly put the household under the $250,000 Additional Medicare Tax threshold, losing $27." -us,scenario_049,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $16,408 premiums as reducing payroll-taxable wages to $236,592. Medicare tax and Additional Medicare Tax both apply to the full $253,000, so it lost $237.92 of Medicare tax and the $27 surtax." -us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used a rough Social Security base of about $180,000 instead of $184,500 and rounded the total down to $14,000. The exact sum is $11,439 + $3,668.50 + $27 = $15,134.50." -us,scenario_049,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 Social Security wage base at $183,600 instead of $184,500. That cost $55.80 of Social Security tax, and it also rounded Medicare tax to $3,669." -us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It said no Additional Medicare Tax applies, which only holds if the $16,408 premiums are subtracted from Medicare wages. Its $14,808 matches 1.45% of $236,592 plus Social Security on a base of about $183,600, instead of the full $253,000 and the $184,500 base." -us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums to get $236,592 of FICA wages. That gave $3,430.58 of Medicare tax instead of $3,668.50 and wrongly removed the $27 Additional Medicare Tax; its $11,439 Social Security figure was right." -us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"It subtracted the $16,408 premiums from Social Security and Medicare wages. That dropped Medicare wages below the $250,000 threshold and removed the $27 surtax. It also used a $183,000 base instead of $184,500." -us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"Its submitted $19,354.50 is 7.65% of the full $253,000, so it ignored the Social Security wage base cap altogether. Its reasoning also wavered between the 2024 and 2025 bases ($168,600 and $176,100) instead of the 2026 base of $184,500." -us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"Its parts ($11,475 + $3,668.50 + $27) add to $15,170.50, but it submitted $23,780.75, a plain addition error. Its $11,475 Social Security figure is also not 6.2% of the 2026 base of $184,500, which is $11,439." +us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning reached the correct $15,134.50 ($11,439 SS on the $184,500 base, plus $3,668.50 Medicare, plus $27 Additional Medicare). It then submitted $14,503.98, a number that none of its stated calculations produce." +us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 wage base of $168,600 instead of $184,500 for 2026. It applied the $200,000 single Additional Medicare threshold instead of the $250,000 joint threshold. It also invented a $904.13 NH state payroll tax, even though NH has none. Its listed parts add to $15,502.83, not the $20,502.83 it submitted." +us,scenario_049,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It named Social Security, Medicare and Additional Medicare Tax as the components but submitted 0. That leaves out the $11,439 SS tax, the $3,668.50 Medicare tax and the $27 Additional Medicare Tax on the spouse's $253,000 wages." +us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That understates SS tax by $520.80. Its Medicare ($3,668.50) and Additional Medicare ($27) figures are correct." +us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning computed the correct $15,134.50 twice. It then submitted $13,164.85, which none of its stated components support." +us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It considered a projected 2026 base but settled on the 2025 base of $176,100 instead of the 2026 base of $184,500. That gives $10,918.20 SS tax instead of $11,439. Its Medicare and $27 Additional Medicare figures are correct." +us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It treated $176,100 as the 2026 Social Security wage base, but that is the 2025 figure. The 2026 base is $184,500, so SS tax is understated by $520.80. The Medicare and Additional Medicare amounts are correct." +us,scenario_049,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a guessed 2026 wage base of $180,900 instead of $184,500. That gives $11,215.80 SS tax instead of $11,439. The Medicare and Additional Medicare components are correct." +us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,408 in employer-sponsored premiums from FICA wages, but payroll tax applies to the full $253,000. This cut Medicare tax to $3,430.58 and wrongly removed the $27 Additional Medicare Tax. It also capped SS at the 2025 base of $176,100 instead of $184,500." +us,scenario_049,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Its total of $14,850.984 equals 1.45% of $236,592 ($3,430.584) plus 6.2% of $184,200. That means it subtracted the $16,408 in premiums from Medicare wages and so dropped the $27 Additional Medicare Tax. It also used a $184,200 wage base instead of $184,500." +us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated the $16,408 in employer-sponsored premiums as pretax, reducing Medicare wages to $236,592. Medicare tax should be on the full $253,000, which also brings in $27 of Additional Medicare Tax. It also capped SS at an estimated $183,900 instead of $184,500." +us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of $184,500 for 2026. Even those stated components add to $14,613.70, not the $13,959.81 it submitted. The correct total is $11,439 + $3,668.50 + $27 = $15,134.50." +us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted the $16,408 in premiums to get FICA wages of $236,592, giving $3,430.58 Medicare tax and no Additional Medicare Tax. Its $14,814 also implies a $183,600 SS cap instead of $184,500. The correct base is the full $253,000, which gives $3,668.50 Medicare tax plus $27 Additional Medicare Tax." +us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It used Medicare wages of $236,592, net of the $16,408 in premiums, instead of the full $253,000. That cut Medicare tax and removed the $27 Additional Medicare Tax. It also used a projected SS cap of $174,900 instead of $184,500." +us,scenario_049,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"Its $4,058.45 is barely above the Medicare tax alone ($3,668.50). That means it essentially left out the $11,439 in Social Security tax (6.2% of the $184,500 wage base) on the spouse's $253,000 wages." +us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages by the $16,408 in premiums to $236,592, giving $3,430.58 Medicare tax and no Additional Medicare Tax. It also capped SS at $181,200 instead of the 2026 base of $184,500." +us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $14,613.34 matches SS capped at the 2025 base of $176,100 plus $3,668.50 Medicare and $27 Additional Medicare. It missed the 2026 base of $184,500, which gives $11,439 in SS tax." +us,scenario_049,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the 2025 wage base of $176,100 instead of the 2026 base of $184,500. That understates SS tax by $520.80. The Medicare and $27 Additional Medicare figures are correct." +us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It gave no parseable payroll_tax value and no explanation, so no answer could be scored against the $15,134.50 total." +us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $19,648 is far above the maximum possible employee FICA of $15,134.50 on $253,000 of wages. That is what you get by taxing wages above the $184,500 Social Security wage base at 6.2% instead of capping SS tax at $11,439." +us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It overstated the total at $15,302. The parameters it used don't match the 2026 values: SS tax capped at $11,439 on the $184,500 base, $3,668.50 Medicare, and $27 Additional Medicare above the $250,000 joint threshold." +us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It excluded the $16,408 in employer-sponsored premiums from FICA wages, leaving $236,592. That cut Medicare tax to $3,430.58 and removed the $27 Additional Medicare Tax. It also capped SS at $183,600 instead of $184,500." +us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"Its SS tax of $11,439 is correct. It figured Medicare tax on wages net of the $16,408 in pretax premiums instead of the full $253,000. That understates Medicare tax by $237.92 and wrongly leaves out the $27 Additional Medicare Tax on wages above $250,000." +us,scenario_049,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used the correct $184,500 SS base but subtracted the $16,408 in premiums to get $236,592 in Medicare wages. Medicare tax applies to the full $253,000 ($3,668.50), and the $3,000 above the joint threshold adds $27 of Additional Medicare Tax." +us,scenario_049,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $16,408 in employer-sponsored premiums as pretax and reduced payroll wages to $236,592. That gives $3,430.58 Medicare tax and no Additional Medicare Tax, instead of $3,668.50 plus $27 on the full $253,000." +us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used an approximate $180,000 SS wage base instead of the 2026 base of $184,500 and rounded the total down to $14,000. The exact components are $11,439 + $3,668.50 + $27 = $15,134.50." +us,scenario_049,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It used a guessed 2026 wage base of $183,600 instead of $184,500. That gives $11,383 SS tax instead of $11,439. Its Medicare and $27 Additional Medicare figures are correct." +us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It wrongly concluded that no Additional Medicare Tax applied. Its $14,808 is consistent with reducing Medicare wages by the $16,408 in premiums and using a roughly $183,500 SS cap. Wages are the full $253,000, so $27 of Additional Medicare Tax applies and SS tax is $11,439." +us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It subtracted the $16,408 in pretax premiums to get FICA wages of $236,592. Medicare tax applies to the full $253,000 of gross wages ($3,668.50), and the $3,000 above the $250,000 joint threshold triggers $27 of Additional Medicare Tax. Its SS tax of $11,439 is correct." +us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"It reduced Social Security and Medicare wages by the $16,408 in premiums and so dropped the $27 Additional Medicare Tax. It also capped SS at an estimated $183,000 instead of the 2026 base of $184,500." +us,scenario_049,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"It subtracted the $4,800 in health insurance premiums to get Medicare wages of $248,200, which put them below the $250,000 joint threshold. That understated Medicare tax and wrongly removed the $27 Additional Medicare Tax. Medicare tax applies to the full $253,000." +us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"It submitted $19,354.50, which equals 6.2% of the full $253,000 plus $3,668.50 Medicare. That ignores the $184,500 Social Security wage base cap, which limits SS tax to $11,439. It also leaves out the $27 Additional Medicare Tax, even though its own reasoning computed capped totals." +us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"Its own components ($11,475 + $3,668.50 + $27) add to $15,170.50, but it submitted $23,780.75, which is an arithmetic error. Its $11,475 SS figure also exceeds the correct $11,439, which is 6.2% of the $184,500 wage base." us,scenario_049,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -4313,45 +4359,46 @@ us,scenario_049,state_income_tax_before_refundable_credits,inkling,llm_error,thr us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It itemized the interest and dividends correctly, subtracted the $4,800 joint exemption to reach $5,021, and then multiplied by a 1% rate it believed applied in 2026. That rate comes from the 2021 phase-down schedule, which the 2023 law replaced. The tax was repealed effective 2025, so the correct 2026 amount is $0, not $50.21." us,scenario_049,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own arithmetic gave $2,641.60, which already taxed the $180 capital gain distribution at 12% instead of 0%, and it then submitted an unrelated $3,086 that none of its steps supports. Correct: $23,900 of ordinary taxable income taxed at 10%/12% = $2,620, plus $0 on the $180." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It ran the $180 of non-Schedule-D capital gain distributions through the ordinary 12% bracket. Those gains get the preferential 0% rate because taxable income is below the 0% LTCG threshold. That overstated tax by $21.60 ($2,641.60 instead of $2,620)." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer insurance premium from the stated $40,000 of wages and used the outdated $15,000 deduction and $11,600 bracket. It then set the tax to $0 by citing the EITC, but the EITC is refundable and excluded from this measure. A childless 25-year-old earning $40,000 is also far above the EITC phase-out. The correct pre-refundable tax is $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It had the right $16,100 standard deduction but never ran the brackets, rounding to $2,700 instead of computing 10% × $12,400 + 12% × $11,500 = $2,620 on the $23,900 of ordinary income. It also treated the $180 capital gain as ordinary income instead of taxing it at 0%." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It computed $2,642, which taxed the $180 capital gain distribution at 12% instead of 0%, and then switched to $2,960 using a $16,000 deduction and a $12,150 bracket. Those figures produce about $2,670, not $2,960, so the final number has no basis in its own work. The correct 2026 parameters ($16,100 deduction, $12,400 bracket) give $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It reached $2,642 with the correct 2026 deduction and brackets, then submitted $2,867 with no supporting calculation. It also taxed the $180 capital gain distribution at 12% instead of the 0% preferential rate, which gives $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It worked out roughly $2,642 with the correct $16,100 deduction and $12,400 bracket, then submitted $2,955 without any arithmetic to support it. Taxing the $180 capital gain distribution at 0% gives $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and $11,925 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400. It also included the $180 capital gain in ordinary income at 12% rather than 0%. Together these overstate tax at $2,783 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used a $12,150 10%-bracket ceiling instead of $12,400 and got $2,646.60. It then inflated that to $3,059 by citing 'adjustments' that do not exist. The $180 capital gain distribution is taxed at 0%, not at ordinary rates, so the tax is $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and the 2025 $11,925 bracket ceiling instead of the 2026 values of $16,100 and $12,400. Although it said the capital gains were taxed at 0%, it ran the full $24,780 (including the $180) through ordinary rates, producing $2,735 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, applying a personal exemption and a 15% bracket. The OBBBA made the TCJA rates permanent with a $16,100 standard deduction and no exemption. It also wrongly subtracted the $5,389 employer insurance premium from the stated $40,000 of wages." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law (an $8,600 standard deduction, a $4,950 personal exemption and a 15% second bracket) instead of the permanent post-OBBBA structure: a $16,100 deduction, no exemption and 10%/12% brackets. It also taxed the $180 gain at ordinary rates. The result is $3,385.75 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 gain at 0%, but it used a $15,400 standard deduction and a $12,200 bracket ceiling instead of the 2026 values of $16,100 and $12,400. That left ordinary income at $24,600 instead of $23,900 and tax at $2,708 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,300 deduction, a $5,150 exemption and a 15% bracket, and it subtracted the $5,389 employer premium from wages that were already stated as $40,000. Its $2,621.15 lands near $2,620 only because these errors offset each other; the correct path is $40,000 − $16,100 = $23,900 at 10%/12%." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It used the outdated $15,000 standard deduction and then subtracted a $1,300 Child Tax Credit, but this childless single filer qualifies for no nonrefundable credits. The correct tax is $2,620, with $0 in credits." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer insurance premium from the stated $40,000 of wages and applied a pre-TCJA standard deduction plus personal exemption of about $13,400. Under 2026 law the only deduction is the $16,100 standard deduction, leaving $23,900 taxed at 10%/12% = $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,300 deduction, a $5,150 exemption and a 15% bracket, and it subtracted the $5,389 employer premium from the stated wages. Its $2,621.15 is only coincidentally close to the correct $2,620, which comes from $23,900 of ordinary income at 10%/12%." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Under 2026 brackets its $2,388 corresponds to about $21,970 of ordinary taxable income, roughly $1,930 below the correct $23,900 ($40,000 − $16,100), which at 10%/12% yields $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer insurance premium from the stated $40,000 of wages and used a $15,700 deduction and the 2025 $11,925 bracket ceiling. That cut taxable income to $19,091 and tax to $2,052.42. The correct figures are a $16,100 deduction and $23,900 of ordinary income, giving $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a pre-TCJA personal exemption and a 15% second bracket for 2026. The OBBBA made the TCJA structure permanent: a $16,100 standard deduction, no exemption and a 12% second bracket. That overstates tax at $3,392 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 gain at 0%, but it assumed the TCJA expired, using an $8,300 deduction, a $5,300 exemption and a 15% bracket. Under current 2026 law ($16,100 deduction, 10%/12% brackets), the $23,900 of ordinary income is taxed at $2,620, not $3,350." -us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used the 2024 $14,600 deduction and the 2025 $11,925 bracket and got $2,831.10. It then submitted $2,530 after an irrelevant discussion of the premium tax credit, a refundable credit that cannot affect this measure. The correct 2026 calculation gives $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $12,200 10%-bracket ceiling instead of the 2026 value of $12,400, and it taxed the $180 capital gain distribution at the ordinary 12% rate instead of 0%. That produces $2,645.60 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out all tax, but the $16,100 deduction against $40,000 of wages leaves $23,900 of taxable income. At the 10%/12% brackets that is $2,620 of tax, with no offsetting nonrefundable credits." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no real derivation. Under 2026 brackets its $2,146 corresponds to about $19,950 of taxable income, roughly $3,950 below the correct $23,900 ($40,000 − $16,100), which yields $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It estimated the 2026 standard deduction at $15,350 and the 10% ceiling at $12,000; the actual values are $16,100 and $12,400. It also taxed the $180 gain at 12% rather than 0%, overstating tax at $2,830.60 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It ran the $180 of non-Schedule-D capital gain distributions through the ordinary brackets. Those distributions are taxed at the 0% preferential rate at this income level, so its answer is $21.60 too high ($2,641.60 instead of $2,620)." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It included the $180 capital gain distribution in the ordinary 12% bracket. That gain is taxed at the 0% preferential rate because taxable income is below the 0% LTCG threshold, so tax is $2,620, not $2,641.60." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It taxed all $24,080 at ordinary 10%/12% rates. The $180 non-Schedule-D capital gain distribution gets the 0% preferential rate, leaving only $23,900 taxed at ordinary rates, for $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $24,680 of taxable income implies a standard deduction of about $15,500 instead of the 2026 value of $16,100. Its $2,781 does not match any 2026 bracket computation on that income. The correct result is $23,900 of ordinary income taxed at $2,620, with $0 on the $180 gain." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and $11,925 bracket ceiling instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 gain at 12% instead of 0%, giving $2,783 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 post-OBBBA $15,750 standard deduction instead of the 2026 value of $16,100, and it taxed the $180 capital gain distribution at 12% rather than 0%. That gives $2,684 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,225 bracket ceiling instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 gain at ordinary rates instead of 0%, producing $2,729.10 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed a reversion to pre-TCJA rules, with an $8,509 deduction, a $5,427 exemption and a 15% bracket. It also subtracted the $5,389 employer premium from the stated $40,000 of wages. Current 2026 law is a $16,100 deduction with 10%/12% brackets, giving $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used inflated bracket ceilings (10% up to $12,750) and then subtracted a fictitious ~$545 'nonrefundable credit for net capital gains'. The 0% rate on the $180 gain only removes the $21.60 it would otherwise owe on that gain, leaving $2,620, not $2,090." -us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied the ordinary 12% rate to the $180 of non-Schedule-D capital gain distributions instead of the 0% preferential rate. That overstates tax by $21.60: $2,641.60 instead of $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $2,618.60 using a slightly wrong $16,300 deduction and $12,350 bracket, then submitted $1,410, which contradicts its own arithmetic and corresponds to no credit or deduction. The correct figure is $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It claimed nonrefundable credits reduce the tax to zero, but a childless single filer with no child-care, education or other qualifying expenses has no nonrefundable credits. The liability on $23,900 of ordinary taxable income is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Its own working reached $2,641.60 by taxing the $180 of capital gain distributions at 12% instead of the 0% long-term rate. It then submitted $3,086, which nothing in its working supports. The correct figure is 10%/12% on $23,900 of ordinary taxable income, which is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It put the $180 of non-Schedule D capital gains into ordinary taxable income ($24,080) and taxed it at 12%. Capital gain distributions get the 0% long-term rate at this income. Ordinary income is $23,900, so the tax is $2,620, not $2,641.60." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted $5,389 of ESI premiums from wages, used a $15,000 standard deduction and 2024-era brackets, and then set the tax to $0 by citing the EITC. The EITC is refundable, so it does not count in this measure, and a childless filer at $40,000 is phased out of it anyway. The correct tax is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It rounded taxable income to about $24,000 and reported a rough $2,700 instead of applying the exact $12,400 10%-bracket edge to $23,900 of ordinary income. It also taxed the $180 of capital gains as ordinary income rather than at 0%. The exact result is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It first computed about $2,642 with the $180 capital gain taxed at 12%. It then switched to a $16,000 deduction and a $12,150 bracket and reported $2,960, which those figures do not produce. The correct 2026 figures ($16,100 deduction, $12,400 bracket, 0% on the gain) give $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It computed $2,642 by including the $180 of capital gain distributions in ordinary income at 12%, then reported $2,867 without supporting it. With the gain at 0%, $23,900 of ordinary income yields $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its own arithmetic gave $1,240 + $1,402 = $2,642, with the $180 gain taxed at 12%, but it reported $2,955 by claiming to apply 2026 brackets, which does not follow. The correct ordinary taxable income of $23,900 produces $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and the 2025 $11,925 10%-bracket edge instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 of capital gains at ordinary rates. Together these overstated taxable income at $25,180 and gave $2,783 instead of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $12,150 10%-bracket edge instead of $12,400 and taxed the $180 gain at 12%, which gave $2,646.60. It then inflated that to $3,059 with no stated basis. The correct figure is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a pre-OBBBA projected $15,400 standard deduction and the 2025 $11,925 bracket edge instead of the 2026 values of $16,100 and $12,400. It said the capital gain was taxed at 0% but still left the $180 in the 12% computation. The result was $2,735 instead of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a pre-TCJA standard deduction plus personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, with a $16,100 deduction, no exemption, and a 12% bracket. It also subtracted $5,389 of ESI premiums that the facts never describe as pre-tax wage reductions." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a TCJA-expiration regime: an $8,600 standard deduction, a $4,950 personal exemption, and a 15% second bracket. 2026 law under OBBBA keeps the $16,100 deduction, no exemption, and 10%/12% brackets, which gives $2,620, not $3,385.75." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 gain at 0%, but it used a pre-OBBBA projected $15,400 standard deduction and a $12,200 10%-bracket edge. The 2026 values are $16,100 and $12,400, which leave $23,900 of ordinary income taxed at $2,620, not $2,708." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired ($8,300 deduction plus $5,150 exemption, 15% bracket) and subtracted $5,389 of ESI premiums from wages as pre-tax. These errors offset each other and landed near the correct figure. The correct 2026 computation is $40,000 − $16,100 = $23,900 at 10%/12%, or $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It used the 2025 $15,000 standard deduction and then subtracted a $1,300 nonrefundable Child Tax Credit. This filer has no children, so no nonrefundable credits apply. The correct tax is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted $5,389 of ESI premiums from wages and applied a pre-TCJA standard deduction plus personal exemption of about $13,400. 2026 law allows a $16,100 standard deduction against the full $40,000 of wages, which leaves $23,900 taxable and a tax of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It treated 2026 as a post-TCJA-expiration year ($8,300 deduction, $5,150 exemption, 15% bracket) and wrongly reduced wages by $5,389 of ESI premiums. The two errors offset and landed $1.15 from the correct $2,620, which comes from the $16,100 deduction and 10%/12% brackets on $23,900." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It reported $2,388 with no derivation, which understates the tax by $232. The correct computation is $40,000 − $16,100 = $23,900 of ordinary taxable income, taxed at $1,240 + 12% × $11,500 = $2,620, with the $180 capital gain at 0%." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $5,389 of ESI premiums from wages as pre-tax deductions and used a $15,700 standard deduction, which understated taxable income at $19,091. The full $40,000 of wages less the $16,100 deduction leaves $23,900, taxed at $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied TCJA-expiration rules (personal exemption and a 15% bracket) to 2026. OBBBA keeps the $16,100 standard deduction and the 10%/12% brackets, which give $2,620 rather than $3,392." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 gain at 0% but assumed the TCJA expired, using an $8,300 deduction, a $5,300 exemption, and a 15% bracket. 2026 law gives a $16,100 deduction and 12% on income above $12,400, for a tax of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used the 2024 $14,600 standard deduction and the 2025 $11,925 bracket edge, which gave $2,831.10. It then submitted $2,530 based on irrelevant premium tax credit reasoning that does not change pre-refundable income tax. The correct 2026 figures give $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It ended the 10% bracket at $12,200 instead of the 2026 edge of $12,400. It also taxed the $180 of capital gain distributions at 12% instead of 0%. Correctly computed, $23,900 of ordinary income yields $2,620, not $2,645.60." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out all tax on $40,180 of income. The 2026 single deduction is only $16,100, which leaves $23,900 of ordinary taxable income and a tax of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave $2,146 with only a vague reference to the deduction and nonrefundable items, and no nonrefundable credits exist for this filer. The correct tax is $1,240 at 10% plus 12% of $11,500, or $2,620, on $23,900 of ordinary taxable income." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It estimated a $15,350 standard deduction and a $12,000 10%-bracket edge instead of the actual 2026 values of $16,100 and $12,400. It also taxed the $180 capital gain at 12%. The result was $2,830.60 instead of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It ran the $180 of non-Schedule D capital gains through the ordinary 12% bracket. Capital gain distributions are taxed at 0% at this income, so only $23,900 is ordinary income, and the tax is $2,620, not $2,641.60." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It included the $180 of capital gain distributions in the 12%-bracket income ($24,080 taxable) instead of applying the 0% long-term rate. That overstated the tax by $21.60 relative to $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It taxed the $180 of non-Schedule D capital gains at the ordinary 12% rate. That gain carries a 0% preferential rate, so ordinary taxable income is $23,900 and the tax is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of roughly $15,500, which made taxable income $24,680, and applied unstated bracket figures to reach $2,781. The 2026 $16,100 deduction with 0% on the $180 gain leaves $23,900 of ordinary income, taxed at $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the 2025 $15,000 standard deduction and the 2025 $11,925 bracket edge instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 gain at 12%. The result was $2,783 instead of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 instead of the 2026 inflation-adjusted $16,100. It also taxed the $180 capital gain at 12% instead of 0%, which overstated taxable income at $24,430 and gave $2,684 instead of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a pre-OBBBA projected $15,400 standard deduction and a $12,225 10%-bracket edge, and it explicitly taxed the $180 gain as ordinary income. The 2026 parameters ($16,100 and $12,400) with 0% on the gain give $2,620, not $2,729.10." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $5,389 of ESI premiums from wages and applied a reverted pre-TCJA deduction, exemption, and 15% bracket. 2026 law applies the $16,100 standard deduction to the full $40,000 of wages with 10%/12% brackets, for a tax of $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a wrong $12,750 10%-bracket edge. It then subtracted about $545 as a supposed capital-gains credit, but applying the 0% rate to the $180 gain only removes 12% × $180 = $21.60. Correct ordinary income of $23,900 gives $2,620, not $2,090." +us,scenario_051,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It correctly put the $180 gain at 0%, but its ordinary-income bracket arithmetic came out $3 high. The exact 2026 figures ($16,100 deduction, $12,400 10% edge) give $1,240 + 12% × $11,500 = $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It taxed the $180 of non-Schedule D capital gains at the ordinary 12% rate inside $24,080 of taxable income. That gain is taxed at 0% at this income level, so the tax is $2,620, not $2,641.60." +us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $16,300 deduction and a $12,350 bracket edge to compute $2,618.60, then submitted an unrelated $1,410. The correct 2026 parameters ($16,100 and $12,400) with the gain at 0% give $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed nonrefundable credits reduce the tax to zero, but a childless single filer with no qualifying expenses has no nonrefundable credits. The tax on $23,900 of ordinary taxable income stands at $2,620." us,scenario_051,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model awarded $560 of childless EITC without applying the phaseout endpoint for a single filer. At $40,000 of earnings, the childless EITC has fully phased out." us,scenario_051,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required output contract." us,scenario_051,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own EITC calculation correctly reached $0 and stated that no other refundable credits apply, but it submitted $2,557.60. The submitted value contradicts its completed derivation and does not represent any applicable refundable-credit component." @@ -4369,100 +4416,102 @@ us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract. us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax plus $580 of Medicare tax and explicitly totaled them as $3,060, but submitted $3,074.95. The submitted value is inconsistent with every component in its explanation." us,scenario_051,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the inflation-adjusted 2026 amount of $12,835, and calculated $830.40. It then submitted $1,020, which does not follow from its own calculation. The correct figure is 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"It applied the 2025 single standard deduction of $12,500 without the 2026 inflation indexing that Act 11 requires, which raises it to $12,835. Taxable income was overstated by $335, so the tax came out at $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"It claimed Louisiana has no individual income tax. Louisiana taxes individual income at a flat 3% on AGI minus the $12,835 single standard deduction, which comes to $820.35 here." -us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"It claimed the standard deduction and personal exemption wipe out all liability. The 2026 Louisiana standard deduction is $12,835 and there is no separate personal exemption, so $27,345 of income is taxed at 3%, giving $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"It cited Act 11 correctly but used the base-year $12,500 single standard deduction and missed the 2026 inflation indexing to $12,835. That gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"It calculated $830.40 with the unindexed $12,500 deduction and then subtracted a made-up 'personal credit' to get $632. The reform eliminated the personal exemption and its credit, and the correct figure is 3% × ($40,180 − $12,835) = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"It started from the unindexed $12,500 standard deduction and then cut the result to $812 with a made-up 'rounding of federal AGI adjustments'. No such adjustment exists. The correct calculation uses the indexed $12,835 deduction and gives $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835, so taxable income came out at $27,680 instead of $27,345 and the tax at $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"It subtracted a federal income tax deduction of $2,783, but Louisiana's 2024 reform repealed that deduction starting in 2025. It also used the unindexed $12,500 standard deduction instead of $12,835, and arrived at $747 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"It calculated $830.40 using the unindexed $12,500 deduction and then submitted $1,080 with no supporting step. The correct calculation uses the 2026 indexed deduction of $12,835 and gives 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"It assumed a $12,500 standard deduction and missed the 2026 inflation adjustment to $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"It combined the 3% flat rate with the pre-reform $4,500 standard deduction and $4,500 personal exemption, which overstated taxable income at $31,180. Under the reform the only deduction is the single standard deduction, $12,835 for 2026, so taxable income is $27,345 and the tax is $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"It applied a 4.25% rate that does not exist in Louisiana law; the 2026 rate is a flat 3%. It also reduced AGI to $34,791 by subtracting the ESI premiums and used the old $4,500 exemption. The correct calculation is 3% × ($40,180 − $12,835) = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835, which overstated tax by $10.05 at $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 indexed standard deduction of $12,835, which gave $27,680 of taxable income and $830.40 of tax instead of $27,345 and $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"It used pre-2025 Louisiana law, with the 1.85%/3.5% brackets, the federal income tax deduction and the $4,500 combined exemption, all of which Act 11 replaced for 2025 and later. It also reduced AGI to $34,791 by subtracting ESI premiums instead of using $40,180. The correct calculation is a flat 3% on $40,180 − $12,835, which is $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"It applied Louisiana's 'progressive' rate schedule, but the graduated brackets were repealed for 2025 and later and replaced by a flat 3% rate. The correct calculation is 3% × ($40,180 − $12,835) = $820.35, not $724." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"It used the repealed 1.85%/3.5% brackets and the old $4,500 exemption, and started from an AGI of $34,791 that wrongly removes the ESI premiums. Under the 2026 rules, the $40,180 AGI minus the $12,835 standard deduction is taxed at a flat 3%, giving $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"It subtracted a federal income tax deduction and a $4,500 exemption and applied the 1.85%/3.5% brackets, all of which the 2024 reform eliminated starting in 2025. Under the 2026 rules, a flat 3% applies to $40,180 minus the $12,835 standard deduction, giving $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At the 3% rate, $1,115 implies about $37,167 of taxable income, which means it effectively left out most of the $12,835 standard deduction. The correct figure is 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-reform $4,500 combined deduction and the 1.85%/3.5% brackets, applied to an AGI wrongly reduced to $34,791 by the ESI premiums. Under the 2026 rules, a flat 3% applies to $40,180 − $12,835, giving $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"It applied the repealed $4,500 standard deduction/exemption and the 1.85%/3.5% graduated brackets. Act 11 replaced these for 2025 and later with a flat 3% rate and a single standard deduction that is $12,835 in 2026, which gives $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-2025 $4,500 deduction and the 1.85%/3.5% rate schedule instead of the reform's flat 3% rate with a $12,835 standard deduction for 2026, which overstated the tax at $1,042.55 against $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"It used the 2025 base amount of $12,500 for the single standard deduction and did not apply the 2026 inflation indexing to $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"It applied the $12,500 standard deduction without the 2026 inflation adjustment to $12,835, which overstated taxable income by $335 and the tax by $10.05." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"It claimed deductions and exemptions eliminate all Louisiana tax. The $12,835 standard deduction leaves $27,345 taxable at 3%, which gives $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"It gave no actual calculation. At 3%, $781 implies about $26,033 of taxable income, which means a deduction of roughly $14,147 instead of the $12,835 single standard deduction. The correct figure is 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"It subtracted the 2025 single standard deduction of $12,500 instead of the 2026 indexed $12,835, so it taxed $27,680 instead of $27,345 and got $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 standard deduction without the 2026 inflation adjustment to $12,835, which overstated the tax at $830.40 against $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"It applied the base-year $12,500 single standard deduction instead of the inflation-indexed 2026 amount of $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction and missed that Act 11 indexes it for inflation, to $12,835 in 2026, so the tax came out at $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"It assumed a $12,500 standard deduction instead of the 2026 indexed $12,835, which overstated taxable income by $335 and the tax by $10.05." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 single deduction instead of the 2026 amount of $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"It applied the 2025 base standard deduction of $12,500 instead of the 2026 inflation-adjusted $12,835, so it taxed $27,680 instead of $27,345 at 3%." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At 3%, $680 implies about $22,667 of taxable income, which means roughly $17,500 of deductions instead of the $12,835 single standard deduction. The correct figure is 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal $15,000 standard deduction instead of Louisiana's own 2026 single standard deduction of $12,835. It then applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate, and got $675 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the 2026 amount of $12,835 and rounded $830.40 to $830. The correct figure is 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 inflation-indexed single standard deduction of $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"It used pre-reform law throughout: a federal income tax deduction, a $4,500 standard deduction plus a $4,500 exemption, and the 1.85%/3.5% brackets. It also cut AGI to $34,791 by subtracting the ESI premiums. Under the 2026 rules, a flat 3% applies to $40,180 − $12,835, giving $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"It applied the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835 and rounded the result to $830. The correct liability is $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this output, so there was no answer to score. The correct calculation is 3% × ($40,180 − $12,835) = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction without the 2026 inflation indexing to $12,835, which gave $830.40 instead of $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"It used the pre-reform $4,500 standard deduction instead of the 2026 single standard deduction of $12,835 and left out the $180 of capital gains. It then submitted $1,208, which does not match its own $1,065 calculation. The correct figure is 3% × $27,345 = $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"It subtracted the 2025 base deduction of $12,500 instead of the 2026 inflation-indexed $12,835, which overstated the tax by $10.05 at $830.40." -us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal $15,750 standard deduction plus a $4,500 Louisiana exemption and applied made-up 1.75%/3.55% brackets. It then submitted $573, which does not match its own $482.52 calculation. Louisiana in 2026 taxes $40,180 minus its own $12,835 standard deduction at a flat 3%, giving $820.35." -us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At 3%, $1,324.80 implies $44,160 of taxable income, which is more than the $40,180 AGI, so it either used a rate above 3% or took no standard deduction. The correct figure is 3% × ($40,180 − $12,835) = $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction to calculate $830.40, then submitted $1,020 without supporting arithmetic. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"The model claimed Louisiana has no individual income tax. Louisiana has a flat 3% individual income tax. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"The model claimed the standard deduction and personal exemption wipe out the liability. The reform eliminated the personal exemption, and the standard deduction does not wipe out this household's liability. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"The model applied the flat 3% rate to income less 2025's $12,500 deduction and obtained $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction to get $830.40. It then subtracted a personal credit that does not exist under the reformed Louisiana tax, which has no personal credit or exemption, and landed on $632. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction to calculate about $830, then made an unsupported downward adjustment for 'AGI rounding' to reach $812; Louisiana AGI is $40,180. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"The model subtracted a $2,783 deduction for federal income tax, which the 2025 flat-tax reform repealed. It also used 2025's $12,500 deduction. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"The model computed $830.40 with the unindexed $12,500 deduction. It then raised the result to $1,080 with no rule behind it; deductions and nonrefundable credits can only lower tax. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"The model paired the new 3% flat rate with the repealed pre-2025 offsets: a $4,500 standard deduction plus a $4,500 personal exemption. The reform replaced those offsets with a single standard deduction. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"The model used a 4.25% rate and a $4,500 exemption instead of the reformed 3% rate and standard deduction, and subtracted the $5,389 ESI premium to cut AGI to $34,791 rather than $40,180. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"The model applied repealed pre-2025 Louisiana law: the federal income tax deduction, the $4,500 combined exemption and the 1.85%/3.5% brackets. It also wrongly reduced AGI to $34,791 by subtracting the ESI premium. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"The model applied progressive Louisiana brackets that the 2025 flat-tax reform replaced. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"The model used the repealed 1.85%/3.5% brackets and the old $4,500 exemption. It also started from an AGI of $34,791 by wrongly subtracting the ESI premium from wages. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"The model deducted federal income tax paid, which the reform repealed, and used the old $4,500 exemption and the 1.85%/3.5% brackets. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"The model gave $1,115 without a derivation. That figure is consistent with too small a deduction or old progressive rates. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"The model applied the repealed 1.85%/3.5% brackets with only a $4,500 deduction. It also wrongly cut AGI to $34,791 by subtracting the ESI premium. The flat 3% on $40,180 − $12,835 gives $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"The model started from the correct $40,180 AGI but applied pre-2025 law: a $4,500 deduction and 1.85%/3.5% brackets. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"The model used the obsolete $4,500 standard deduction and the 1.85%/3.5% bracket schedule, which the 2025 reform replaced. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions fully offset the tax, but they leave positive Louisiana liability on $40,180 of AGI. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"The model gave $781 with no actual computation. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"The model applied the unindexed $12,500 single deduction. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"The model's estimated $12,500 deduction is the 2025 base amount. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 standard deduction. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"The model gave $680 without a derivation. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the federal standard deduction rather than Louisiana's own deduction and applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction and then rounded to $830. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"The model applied repealed pre-2025 law throughout: a deduction for federal tax, a $4,500 standard deduction plus a $4,500 exemption, and 1.85%/3.5% brackets. It also cut AGI to $34,791 by subtracting the ESI premium. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction and rounded to $830. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or reasoning for this output. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"The model used the obsolete $4,500 deduction and left the $180 of capital gains out of income. Its own arithmetic gave $1,065, yet it submitted $1,208. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,muse-wonder,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the federal standard deduction and a repealed $4,500 Louisiana exemption, then applied made-up graduated rates. Its own figure was $482.52, yet it submitted $573. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." +us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"The model gave $1,324.80 with no derivation. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994." us,scenario_051,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $117,285, which does not match its own figure of $107,277. That figure was also too high: it used a $32,600 standard deduction instead of $32,200, mis-applied the 2026 joint brackets (ordinary tax of $106,932 instead of about $103,866), and did not deduct the $43.28 IRA contribution." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43.28 traditional IRA contribution under the §219(g) active-participant phase-out, so its AGI was $504,821.84 instead of the reference's $504,778.56. At the 32% rate, that $43.28 accounts for the whole $13.89 gap. Its standard deduction ($32,200), brackets, 15% dividend tax and $218.42 NIIT all match the reference." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $504,779 was right, but it used a $28,700 standard deduction instead of the 2026 OBBBA $32,200 and estimated tax with a rough flat ~23% rate instead of the 2026 joint brackets. Its final $103,241 does not follow from any of its own figures, and it never added the $127.20 qualified-dividend tax or the $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess of $105,000 without working through the brackets. The correct calculation taxes $471,730.56 of ordinary income at the 2026 joint brackets ($103,865.78), then adds $127.20 on qualified dividends and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax, which is a payroll tax that the question puts under payroll_tax, to income tax. It also used outdated bracket guesses ($24,150, $98,200, $209,400, $400,000) and a $32,300 standard deduction, then raised its $107,040 subtotal to $110,997 without any computation." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It mentioned the $926 401(k) deferral but did not subtract it, giving an AGI of $505,748 instead of $504,778.56. It also used a $32,300 standard deduction, understated the 2026 joint bracket thresholds ($401,950 instead of $403,550 for the top of the 24% bracket, among others), and left out the $218.42 NIIT owed because MAGI exceeds $250,000." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own bracket estimates applied to about $471,350 of ordinary income give roughly $103,800, but it reported $141,500, an arithmetic error of about $37,000. It also left out the $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax to income tax, although that tax belongs under payroll tax. It also used the outdated pre-OBBBA $30,000 standard deduction instead of $32,200 and inflated the bracket thresholds with its own estimates." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the annual $926 401(k) and $43 IRA contributions as weekly amounts and multiplied them by 52, which created a $31,000 401(k) deduction and a $2,236 IRA deduction. It then raised its $94,754 result to $121,500 without any calculation." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43.28 traditional IRA contribution under the §219(g) active-participant phase-out, so its AGI was $504,822 instead of $504,778.56. At the 32% rate this accounts for the roughly $14 gap. Its standard deduction, brackets, 15% dividend tax and NIIT all match the reference." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,000 standard deduction instead of $32,200 and reported $119,769 of ordinary tax, about $15,000 more than the 2026 joint brackets produce on roughly $474k. It also calculated NIIT on the gross $8,748 instead of $5,748, which is net of the $3,000 allowable capital loss." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages, but only the 401(k) deferral reduces employment income, which is $499,073.84. It also used a $30,800 standard deduction instead of $32,200, skipped the $43.28 IRA deduction, and calculated NIIT on $8,748 instead of the net $5,748." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the TCJA-sunset rules: a $16,900 standard deduction, personal exemptions reduced by the phase-out, and pre-2018 brackets. OBBBA made the TCJA rates, the $32,200 joint standard deduction and the zero exemption permanent for 2026. It also calculated NIIT on $8,748 instead of $5,748." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA had expired and used a $16,700 standard deduction plus $10,600 of personal exemptions with pre-2018 rates. OBBBA keeps the $32,200 standard deduction, no exemptions, and a 32% top rate in this income range. It also calculated NIIT on $8,748 instead of $5,748." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It made up $45,620 of itemized deductions, including mortgage interest and state and local taxes, even though only a mortgage balance was given and Texas has no income tax. It also applied the Pease limitation and pre-TCJA rates and taxed qualified dividends at 20% instead of using the $32,200 standard deduction, OBBBA brackets and the 15% rate." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It said capital losses were capped at $3,000 but reduced AGI by about $28,926 of losses, reaching roughly $476,323 instead of $504,778.56. Its $139,198 result is also far above what the 2026 joint brackets produce on any taxable income near $470k." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no figures. Its $123,753 is about $19,500 above the correct amount, in line with the other models that used the expired pre-TCJA rate schedule and smaller standard deduction and exemptions instead of OBBBA's permanent 2026 brackets and $32,200 standard deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used post-TCJA-expiration rules: a $15,300 standard deduction, $10,100 of personal exemptions and pre-2018 brackets, giving $122,573.78 of ordinary tax. It also subtracted the $16,208 ESI premiums from wages, lowering AGI to $488,614." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It claimed taxable income was above the top marginal bracket, but $472,578.56 falls in the 32% bracket, which runs to $512,450 for joint filers in 2026. It also counted only the head's $7,200 of interest. Its unsupported $105,744 is $1,533 too high." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It reported $113,421 without showing the brackets or deduction it used, about $9,200 above the correct amount. The correct result applies the 2026 OBBBA joint brackets to $472,578.56 of taxable income ($103,865.78) and adds $127.20 of dividend tax and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages, giving an AGI of $488,614, and its regular tax of about $117,547 reflects pre-TCJA rates instead of the 2026 OBBBA brackets." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly used TCJA-sunset rules (personal exemptions, the smaller pre-2018 standard deduction and pre-2018 brackets) and also treated the $16,208 ESI premiums as reducing wages. OBBBA made the TCJA structure permanent, so the correct base is the $32,200 standard deduction applied to an AGI of $504,778.56." -us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA had expired and used a $15,900 standard deduction, partially phased-out exemptions, and the 10/15/25/28/33/35% brackets, which gave $131,672.30 of ordinary tax. It also left out the $218.42 NIIT. The 2026 OBBBA joint schedule with the $32,200 deduction gives $103,865.78 of ordinary tax." -us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It overstated AGI at about $505,800 and used a $32,400 standard deduction. It treated qualified dividends as a $144 reduction instead of $127.20 of tax at 15%, left out the $218.42 NIIT, and rounded to $105,000." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $50,504 is less than half the correct liability. On $471,730.56 of ordinary taxable income, the 2026 joint brackets alone produce $103,865.78, so it badly mis-applied the rate schedule." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It claimed retirement contributions and health premiums wipe out the liability, but those amounts total under $25,000 against $500,000 of wages. Taxable income remains $472,578.56, and the tax is $104,211.41." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 3.8% Net Investment Income Tax ($218.42 on $5,748 of net investment income, since MAGI exceeds the $250,000 joint threshold), which is part of income tax before refundable credits. It also skipped the $43.28 IRA deduction, so its $104,007 reflects only regular tax and the qualified-dividend tax." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $848 of qualified dividends at 20% ($169.60) instead of 15% ($127.20), even though taxable income is far below the 2026 joint 20% threshold of $613,700. It also left out the $218.42 NIIT and skipped the $43.28 IRA deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It calculated NIIT on the gross $8,748 of interest and dividends ($332.42) instead of $5,748, which is net of the $3,000 allowable capital loss ($218.42). It also skipped the $43.28 IRA deduction, so it ended $127.89 too high." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $57,891 is about $46,000 too low. The 2026 joint brackets give $103,865.78 of ordinary tax on $471,730.56, so it badly mis-applied the rate schedule, and it also left out the $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 ESI premiums as a pre-tax reduction of wages, reaching an AGI of $488,614. Only the $926.16 401(k) deferral comes out of employment income ($499,073.84), and the $43.28 IRA contribution is deducted, so AGI is $504,778.56." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43.28 traditional IRA contribution, keeping AGI at $504,821.84 instead of $504,778.56. At the 32% rate that accounts for the whole $13.89 gap. Its brackets, 15% dividend tax and $218.42 NIIT match the reference." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 3.8% NIIT ($218.42 on $5,748 of net investment income, with MAGI far above $250,000) and skipped the $43.28 IRA deduction. Its $104,006.88 is regular tax plus dividend tax on $472,622 instead of $472,578.56 of taxable income." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 ESI premiums as pre-tax wage reductions, reaching an AGI of $488,614. Only the $926.16 401(k) deferral reduces employment income and the $43.28 IRA contribution is deducted, so AGI is $504,778.56 and taxable income is $472,578.56." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rough guess of $140,000 without applying any brackets, about $35,800 too high. The 2026 joint brackets on $471,730.56 give $103,865.78, plus $127.20 of dividend tax and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used projected post-TCJA rules (a $16,600 standard deduction, personal exemptions and pre-2018 rates) instead of OBBBA's permanent 2026 schedule. It also subtracted the $16,208 ESI premiums from wages and calculated NIIT on $8,748 instead of the net $5,748." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,750 joint standard deduction instead of the 2026 OBBBA $32,200 and skipped the $43.28 IRA deduction. This overstated taxable income by about $1,494, which at 32% explains most of its $545 overstatement." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA had expired and used a $16,950 standard deduction, $5,184 of remaining personal exemptions and pre-2018 rates, which gave $125,993 of regular tax. OBBBA keeps the $32,200 deduction, no exemptions and the TCJA brackets for 2026." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages, reaching an AGI of $488,571, and used a $30,750 standard deduction instead of $32,200. It also calculated NIIT on the gross $8,748 instead of $5,748, which is net of the $3,000 allowable capital loss." -us,scenario_052,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It miscalculated ordinary tax as about $104,035. Under the 2026 joint brackets, its own $471,774 of ordinary income gives $82,048 + 32% × $68,224 = $103,879.68, a $155 arithmetic error. It also skipped the $43.28 IRA deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It imputed an $88,400 FLSA overtime premium from the 60-hour schedule and claimed a $4,517.80 qualified-overtime deduction, but a $500,000 earner is FLSA-exempt as highly compensated, so no overtime deduction applies. It also left out the $218.42 NIIT and skipped the $43.28 IRA deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It switched between a $16,100 and a $30,800 standard deduction instead of using the 2026 OBBBA $32,200, and reported taxable income of about $488,648 instead of $472,578.56. It never calculated the $218.42 NIIT or showed the 15% dividend layer." -us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income ($472,579) and ordinary tax ($103,866) match the reference, but it taxed the $848 of qualified dividends at 0%. Taxable income is far above the 2026 joint 0% ceiling of $98,900, so the 15% rate applies ($127.20). It also left out the $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $29,922 net capital loss from ordinary income, ignoring the §1211(b) $3,000 limit, and taxed qualified dividends at 0%. It then reported $165,473.58, which does not match its own $85,184.10 calculation." -us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 35% rate starting at about $256,800, roughly the single-filer threshold. For 2026 joint filers the 32% bracket runs to $512,450, so 35% does not apply here. It also used a $30,000 standard deduction, did not subtract the 401(k) deferral, subtracted the capital loss a second time, and left out NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own working reached $107,277, but it submitted $117,285, an unexplained jump of about $10,000. The $107,277 is also too high: it used a $32,600 standard deduction instead of $32,200, and it estimated $106,932 of tax on $471,374 of ordinary income. The 2026 brackets (24% up to $403,550, 32% above) give about $103,750 on that amount." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It correctly disallowed the $43 traditional IRA deduction under the active-participant phase-out, leaving rounded AGI at $504,822 and taxable income at $472,622. The frozen reference incorrectly deducts $43.28, giving taxable income of $472,578.56; the $43.44 difference also includes $0.16 from rounding the 401(k) deferral. Its $104,225.30 is $0.04 above the exclusion's recorded corrected value, $104,225.26. Its brackets, standard deduction, $127.20 QD tax, and $218.42 NIIT all match." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used a $28,700 MFJ standard deduction instead of the 2026 amount of $32,200, never actually ran the 2026 bracket schedule, and left out the $218.42 NIIT. It then swapped its $109,223 estimate for an unexplained $103,241. Its AGI of $504,779 matches, so the whole error is in the deduction and tax steps." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It submitted a round $105,000 without doing any bracket calculation. The frozen reference uses taxable income of $472,578.56 and ordinary tax of $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT. It incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26, so the round guess is about $775 above that value." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax to income tax, but that 0.9% wage surtax belongs in payroll_tax, not in federal income tax before refundable credits. It then raised its own $107,040 total to $110,997 with no supporting calculation. It also used guessed thresholds (24% ending at $400,000) and a $32,300 standard deduction instead of $403,550 and $32,200." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its reported AGI of $505,748 fails to reflect the $926 traditional 401(k) deferral in its arithmetic. Disallowing the $43 IRA deduction agrees with the exclusion's active-participant phase-out correction, which gives AGI of $504,821.84; $504,778.56 is the frozen reference's AGI, including the defective IRA deduction. It used a $32,300 standard deduction instead of $32,200 and guessed bracket thresholds (24% up to $401,950 instead of $403,550). It also left out the $218.42 NIIT on $5,748 of net investment income." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It got taxable income roughly right (about $472,200) but reported $141,500 of tax, a bracket-arithmetic error of about $37,000. Under the 2026 MFJ schedule, about $471,350 of ordinary income produces roughly $103,740 of tax. It also left out the $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added $2,250 of Additional Medicare Tax (Form 8959) to federal income tax before refundable credits, but that wage surtax belongs in payroll_tax. It also used a $30,000 standard deduction instead of the 2026 amount of $32,200 and its own estimated bracket thresholds, which pushed taxable income up to $474,822." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It read the $926 traditional 401(k) and $43 IRA contributions as weekly amounts and multiplied them by 52, giving $31,000 (after the cap) and $2,236, which cut AGI to $472,512. It then dropped its own $94,754 result and submitted an arbitrary $121,500. The contributions are annual totals: the $926.16 401(k) deferral reduces income, while the $43.28 IRA deduction is fully phased out under the exclusion's correction. Taxable income of $472,578.56 belongs to the frozen reference, which incorrectly takes the IRA deduction; the recorded corrected tax is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the traditional IRA deduction as fully phased out, which left taxable income at about $472,622. The reference deducts the $43.28 IRA contribution, giving taxable income of $472,578.56, and that $43.44 at 32% accounts for the whole $14 gap. Its other components ($32,200 standard deduction, 2026 brackets, $127 QD tax, $218 NIIT) match." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It reported $119,769 of ordinary tax on $474,822 of taxable income, about $15,000 more than the 2026 MFJ brackets produce at that income (about $104,600). It also used a $30,000 standard deduction instead of $32,200. It applied NIIT to $8,748 instead of $5,748, leaving out the $3,000 allowable capital loss that reduces net investment income." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premium from wages, which cut AGI to $488,614. The reference reduces the $500,000 of wages only by the $926.16 401(k) deferral, giving AGI of $504,778.56. It also used a $30,800 standard deduction instead of $32,200 and applied NIIT to $8,748 instead of $5,748." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA law to 2026: a $16,900 standard deduction, personal exemptions, and the old rate schedule. OBBBA made the TCJA brackets and the larger standard deduction permanent ($32,200 MFJ for 2026) and kept personal exemptions at zero. It also applied NIIT to $8,748 instead of $5,748, which is the amount after the $3,000 capital-loss offset." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset and used a $16,700 standard deduction plus $10,600 of personal exemptions with the pre-TCJA rates. Under OBBBA, 2026 has a $32,200 MFJ standard deduction, no personal exemptions, and 24%/32% brackets for this income. It also applied NIIT to $8,748 instead of the $5,748 left after the $3,000 capital loss." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired and itemized $45,620 of deductions, including mortgage interest and state/local taxes that are not in the inputs: only a mortgage balance is listed, with no interest paid, and Texas has no income tax. It also applied the Pease limitation and a 20% dividend rate. OBBBA's 2026 rules give a $32,200 standard deduction, 24%/32% brackets, and a 15% rate on the $848 of qualified dividends." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of about $476,323 means it subtracted nearly the whole capital loss instead of the $3,000 allowed by §1211(b). Even so, its $139,198 of tax is far above what the 2026 brackets produce at that income. The frozen reference's steps are AGI of $504,778.56, taxable income of $472,578.56, and $103,865.78 of ordinary tax, plus $127.20 QD tax and $218.42 NIIT. That calculation incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $123,753 is about $19,500 too high, which fits taxing this income under the pre-TCJA schedule (15%/25%/28%/33% with a standard deduction of about $16,600) on a sunset assumption. Under OBBBA's 2026 schedule, taxable income of $472,578.56 falls in the 24%/32% brackets, giving $103,865.78 plus $127.20 QD tax and $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used TCJA-sunset rules (a $15,300 standard deduction, $10,100 of personal exemptions, and pre-TCJA rates) instead of OBBBA's permanent 2026 parameters ($32,200 standard deduction, no exemptions, 24%/32% brackets). It also subtracted the $16,208 ESI premium from wages, which lowered AGI to $488,614 instead of $504,821.84 under the exclusion's IRA phase-out correction; $504,778.56 is the frozen reference's AGI, including the defective $43.28 IRA deduction." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the head's $7,200 of interest and left out the spouse's $700. It also claimed taxable income was above the top bracket, but the 37% bracket starts at $768,700 MFJ in 2026, and $472,578.56 of taxable income tops out in the 32% bracket. Its $105,744 has no supporting bracket calculation and is about $1,533 above $103,865.78 + $127.20 + $218.42." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It listed the right pieces (wages, interest, dividends, the $3,000 loss limit, and NIIT) but showed no bracket arithmetic, and the 2026 MFJ schedule does not produce its $113,421. The frozen reference uses taxable income of $472,578.56 and ordinary tax of $103,865.78, plus $127.20 at 15% on qualified dividends and $218.42 NIIT. That calculation incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26, still about $9,200 below its answer." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premium from wages, which lowered AGI to $488,614 instead of $504,821.84 under the exclusion's IRA phase-out correction. It then applied higher pre-OBBBA rates, so its regular tax came to about $117,547. The frozen reference's AGI of $504,778.56 and taxable income of $472,578.56 include the defective IRA deduction; its ordinary tax is $103,865.78 plus $127.20 QD tax. The exclusion's recorded corrected total is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a TCJA sunset, with personal exemptions and the pre-TCJA standard deduction and brackets, and it subtracted the ESI premium as a pre-tax deduction. OBBBA's permanent 2026 rules use a $32,200 standard deduction with no exemptions. Wages are reduced by the $926.16 401(k) deferral. The frozen reference's taxable income of $472,578.56 also includes an incorrect $43.28 IRA deduction; the exclusion's IRA phase-out correction disallows it and records tax of $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired and used a $15,900 standard deduction, phased-out personal exemptions, and the 10/15/25/28/33/35% schedule. OBBBA made the $32,200 standard deduction and the 24%/32% brackets permanent for 2026. It also left out the $218.42 NIIT on $5,748 of net investment income." +us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It overstated AGI at about $505,800; after the $926.16 401(k) deferral and $3,000 capital-loss deduction, AGI is $504,821.84 under the exclusion's IRA phase-out correction. The frozen reference's AGI of $504,778.56 incorrectly includes the $43.28 IRA deduction. It used a $32,400 standard deduction instead of $32,200 and rounded ordinary tax up to about $105,200, compared with the frozen reference's $103,865.78; the recorded corrected total tax is $104,225.26. It also left out the $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave $50,504 with no bracket arithmetic, which is less than half the liability. The frozen reference's $472,578.56 of taxable income includes $848 of qualified dividends, leaving $471,730.56 of ordinary income. Its ordinary tax is $82,048 through $403,550 plus 32% of $68,180.56, or $103,865.78; its displayed QD tax and NIIT are $127.20 and $218.42, and its stored total rounds to $104,211.41. That reference incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It reported zero tax on the theory that retirement contributions and health premiums wipe out the liability. Against about $508,748 of gross income, the $926.16 401(k) deferral, $3,000 capital-loss deduction, and $32,200 standard deduction apply; the $43.28 IRA deduction is fully phased out under the exclusion's correction. The frozen reference incorrectly takes that IRA deduction, giving taxable income of $472,578.56 and tax of $104,211.41; the recorded corrected tax is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It reported only regular tax and left out the $218.42 NIIT included in the frozen reference; the exclusion separately flags whether this output includes NIIT as an unlisted input. It correctly disallowed the $43 IRA deduction, giving rounded taxable income of $472,622; the frozen reference's $472,578.56 incorrectly includes that deduction. The exclusion's recorded IRA-corrected value is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $848 of qualified dividends at 20% ($169.60) instead of 15% ($127.20); the 20% rate starts only at $613,700 of MFJ taxable income in 2026. It left out the $218.42 NIIT and disallowed the $43 IRA deduction, using $472,622 of taxable income instead of $472,578.56." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It calculated NIIT as about $332 on $8,748 of interest plus dividends and left out the $3,000 allowable capital loss. That loss reduces net investment income to $5,748, so the NIIT is $218.42. It correctly disallowed the $43 IRA deduction under the exclusion's active-participant phase-out correction; the frozen reference incorrectly takes that deduction." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $57,891 without bracket arithmetic, about 55% of the liability. Taxable income of $472,578.56 under the 2026 MFJ schedule gives $103,865.78 of ordinary tax, plus $127.20 at 15% on the qualified dividends and $218.42 of NIIT, which its total also leaves out." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 employer-sponsored insurance premium as a pre-tax wage reduction, which cut AGI to $488,614 and taxable income to $456,414. The reference reduces wages only by the $926.16 401(k) deferral and deducts the $43.28 IRA contribution, giving AGI of $504,778.56 and taxable income of $472,578.56." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the 401(k) deferral and capital loss and omitted the IRA deduction, which agrees with the exclusion's active-participant phase-out correction. Its rounded taxable income of $472,622 is $43.44 above the frozen reference's $472,578.56; that difference includes the defective $43.28 IRA deduction and $0.16 of input rounding. Its $104,225.30 is $0.04 above the exclusion's recorded corrected value, $104,225.26. Its QD tax and $218 NIIT match." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It stopped at regular tax plus the 15% tax on qualified dividends ($104,006.88) and left out the $218.42 NIIT on $5,748 of net investment income. It also disallowed the $43.28 IRA deduction, so taxable income was $472,622 instead of $472,578.56." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 employer-sponsored insurance premium as pre-tax, which cut AGI to $488,614. The reference reduces wages only by the $926.16 401(k) deferral, giving AGI of $504,778.56 and taxable income of $472,578.56 after the $32,200 standard deduction." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round $140,000 guess without computing taxable income or applying brackets. Taxable income of $472,578.56 is taxed at no more than 32% under the 2026 MFJ schedule, giving $103,865.78, plus $127.20 QD tax and $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used projected post-TCJA parameters ($16,600 standard deduction, personal exemptions, and pre-TCJA rates) instead of OBBBA's permanent 2026 schedule with a $32,200 standard deduction. It also subtracted the $16,208 ESI premium from wages and applied NIIT to $8,748 instead of the $5,748 left after the $3,000 capital loss." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,750 MFJ standard deduction instead of the 2026 amount of $32,200, which added $1,450 of taxable income at 32%. It also disallowed the $43.28 IRA deduction, raising taxable income to $474,072 instead of $472,578.56." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $504,779 matches the frozen reference, which incorrectly takes the $43.28 IRA deduction; the exclusion's IRA phase-out correction gives AGI of $504,821.84. It then applied TCJA expiration: a $16,950 standard deduction, $5,184 of personal exemptions, and pre-TCJA rates. OBBBA kept the $32,200 MFJ standard deduction and 24%/32% brackets for 2026. The frozen reference's taxable income of $472,578.56 and ordinary tax of $103,865.78 reflect the defective IRA deduction; the recorded corrected total tax is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premium from wages, which cut AGI to $488,571. It used a $30,750 standard deduction instead of $32,200 and applied NIIT to $8,748 instead of the $5,748 left after the $3,000 capital loss. Wages are reduced only by the $926.16 401(k) deferral." +us,scenario_052,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It reported $104,035 of ordinary tax on $471,774 of ordinary income, but the 2026 brackets ($82,048 through $403,550, then 32%) give $103,879.68, about $155 less. It also disallowed the $43.28 IRA deduction, so taxable income was $472,622 instead of $472,578.56." +us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so the answer could not be parsed." +us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $4,517.80 OBBBA qualified-overtime deduction by inferring overtime pay from the 60-hour schedule. The engine allows no overtime deduction for this $500,000-salaried head, so taxable income stays at $472,578.56. It also left out the $218.42 NIIT on $5,748 of net investment income." +us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It mixed up two standard deduction figures ($16,100 and $30,800, versus the 2026 MFJ amount of $32,200), gave a taxable income of about $488,648 that conflicts with its own adjustments, and asserted $104,050 without bracket arithmetic. It never included the $218.42 NIIT or the $127.20 QD tax as separate items." +us,scenario_052,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"It calculated NIIT as about $332 on $8,748 of interest plus dividends and left out the $3,000 allowable capital loss. That loss reduces net investment income to $5,748, so the NIIT is $218.42. It correctly disallowed the $43.28 IRA deduction under the exclusion's active-participant phase-out correction. The $43.44 difference from the frozen reference's taxable income also includes input rounding; the exclusion's recorded corrected tax is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI ($504,779) and taxable income ($472,579) match, but it taxed the $848 of qualified dividends at 0%. The 0% rate ends at $98,900 of MFJ taxable income in 2026, so 15% applies ($127.20). It also left out the $218.42 NIIT, so it reported only the $103,866 of ordinary bracket tax." +us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the full $29,922 net capital loss instead of the $3,000 allowed under §1211(b), and it ignored the 401(k) and IRA deductions. It then submitted $165,473.58, which contradicts its own $85,184.10 calculation. It also stretched the 32% bracket and taxed the dividends at 0%." +us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $219,000 at 35%, but in 2026 the MFJ 35% bracket starts at $512,450, so all of this income falls at 32% or below. It also used a $30,000 standard deduction instead of $32,200, did not subtract the $926.16 401(k) deferral, counted the $3,000 capital-loss deduction twice as a tax reduction, and left out the $218.42 NIIT." us,scenario_052,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_052,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified benchmark rule making a 59-year-old Medicare eligible. Medicare's standard age threshold is 65, and the prompt supplies no disability, end-stage renal disease, or other alternative eligibility basis, so the head is not eligible." us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"The model's reasoning explicitly reached the correct $20,939: $11,439 Social Security, $7,250 Medicare and $2,250 Additional Medicare. It then submitted $12,820.40, which matches neither that total nor its own $20,558.11 alternative that excluded the insurance premiums. The submitted value is disconnected from any calculation the model showed." @@ -4498,41 +4547,42 @@ us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model computed us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning totals $20,418.20, using the outdated $176,100 wage base instead of $184,500. It then submitted $32,846, which none of its stated components support." us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model counted the $2,250 Additional Medicare Tax twice: it added it again to a $20,418.20 subtotal that already included it. It also used the outdated $176,100 Social Security wage base instead of the 2026 base of $184,500." us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified rule that makes a 55-year-old Medicare eligible. The spouse is below the age-65 threshold and has no listed disability or other qualifying Medicare pathway, yielding spouse_medicare_eligible = 0." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Subtracted the $3,389 ESI premium input from wages and also took a $950 OBBBA auto-loan-interest deduction, even though no qualifying new-vehicle loan facts are listed. Wages of $66,968.68 are AGI with no adjustments. The model then submitted $6,224, which does not match its own computed $5,335.60." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Took the $950 auto loan interest as an OBBBA qualified passenger vehicle loan interest deduction. That deduction requires a loan for a new, US-assembled vehicle originated after 2024, and none is listed here. Without it, taxable income is $50,868.68 and $468.68 falls in the 22% bracket, giving $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Subtracted the $3,389 ESI premiums from wages and used the 2024 standard deduction ($14,600) and 2023 bracket figures instead of 2026's $16,100 deduction and the $12,400/$50,400 thresholds. Its bracket arithmetic produced $12,761.60, and it then submitted an unrelated $5,744." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Treated the $3,389 ESI premium input as a pre-tax reduction to wages, cutting taxable income to $47,480 instead of $50,868.68. It then understated the bracket tax even on that base: $47,480 yields $5,449.60, not $4,700." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a $16,600 standard deduction instead of the 2026 single amount of $16,100, which kept taxable income just under the 22% bracket. It then arbitrarily raised its own $5,796 result to $6,196 without any computation." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Computed the correct $5,903 from $50,869 taxable income and the 2026 $12,400/$50,400 thresholds. It then discarded that result and submitted an unexplained $6,027 by appealing to 'rounding to bracket estimates'." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Correctly derived taxable income of about $50,869 and tax of about $5,903 under the 2026 brackets. It then replaced that with an unsupported $6,379 attributed to a vague '2026 brackets estimate'." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Used a guessed $15,700 standard deduction and 2025 bracket thresholds ($11,925/$48,475) instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. This pushed $2,794 into the 22% bracket instead of $468.68." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Got taxable income of $50,869 right but built brackets by inflating the 2025 thresholds 2% instead of using the actual 2026 thresholds ($12,400 and $50,400). It also reported $6,154, more than even those approximated brackets produce." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used a $15,650 standard deduction and bracket thresholds of $12,200/$49,950 instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. This overstated taxable income by $450 and taxed $1,369 at 22% instead of $468.68." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Subtracted the $3,389 ESI premiums from wages to get a $63,580 AGI and used a $15,400 standard deduction and a $12,225 10% bracket. The correct figures are wage AGI of $66,968.68, the $16,100 deduction, and a $12,400 10% bracket." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the TCJA sunset in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. The OBBBA made TCJA permanent, so 2026 has a $16,100 standard deduction, no exemption, and 10%/12%/22% brackets." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed the TCJA expired and used an $8,300 deduction plus a $5,300 exemption with 15%/25% brackets instead of the permanent OBBBA amounts ($16,100 deduction and 10%/12%/22% brackets). It also wrongly subtracted the $3,389 ESI premiums from wages." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Used a $15,300 standard deduction instead of $16,100. Its $3,939 is far below what even its own $51,669 taxable income yields under 2026 brackets ($6,079), so it misapplied the bracket rates on top of the deduction error." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Assumed TCJA expiration and applied a pre-TCJA standard deduction, personal exemption, and 15%/25% brackets instead of 2026 OBBBA law ($16,100 deduction and 10%/12%/22% brackets). It also wrongly subtracted the $3,389 ESI premiums from wages." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied post-sunset parameters ($8,300 deduction, $5,300 exemption, 15%/25% brackets) even though the OBBBA made TCJA rates and the $16,100 2026 standard deduction permanent. It also reduced AGI by the $3,389 ESI premium input, which is not a wage reduction here." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no computation. The correct path is $66,968.68 wages minus the $16,100 deduction, which is $50,868.68 taxable, giving $5,903.11. Its $3,737 matches taxable income of only about $33,200 under 2026 brackets, so it understated taxable income or the bracket rates by roughly $17,000 of base." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $3,389 ESI premiums from wages, cutting taxable income to $47,480 instead of $50,868.68, and also slipped arithmetically ($47,480 gives $5,449.60, not $5,459.10). Wages are AGI as given, which puts $468.68 in the 22% bracket." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Treated the $3,389 ESI premium input as a pre-tax reduction, so it applied the correct 2026 deduction and brackets to $47,480 instead of $50,868.68 taxable income. It never reached the 22% bracket." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the pre-OBBBA 2025 standard deduction ($15,000) and 2025 bracket thresholds ($11,925/$48,475) for 2026 instead of $16,100 and $12,400/$50,400. This overstated taxable income and the amount taxed at 22%." -us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied 2024 parameters (a $14,600 standard deduction and $11,600/$47,150 thresholds) instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. That taxed $5,219 at 22% instead of $468.68." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Subtracted a 'basic CTC/nonrefundable credit' for a single 25-year-old with no children or dependents, who qualifies for no nonrefundable credit. Tax of $5,903.11 on $50,868.68 taxable income should stand unreduced." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Gave no computation. The correct path is wages minus the $16,100 deduction, which is $50,868.68 taxable, giving $5,903.11 under the 10%/12%/22% brackets. Its $5,118 matches only about $44,700 of taxable income, so it understated the base or the bracket rates by roughly $6,000 of taxable income." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used a $16,000 standard deduction instead of $16,100 and misapplied the 2026 brackets. Its own $50,969 taxable income yields $5,925.18 under the $12,400/$50,400 thresholds, not $5,793." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Reached the correct $50,869 taxable income but misapplied the bracket thresholds. $5,985 implies the 22% bracket starting near $49,600 instead of $50,400, which overstates the 22% slice above the correct $468.68." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Subtracted $3,389 of ESI premiums from gross wages as pre-tax contributions, giving $47,480 taxable instead of $50,868.68. That removed the $468.68 taxed at 22% and lowered the 12% slice." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Used a roughly $15,000 standard deduction instead of 2026's $16,100, giving about $52,000 taxable instead of $50,868.68. The resulting $5,850 is a loose approximation that does not follow the 2026 bracket arithmetic, under which even $52,000 would yield about $6,250." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed a TCJA sunset with an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets. Under the OBBBA, 2026 law keeps a $16,100 standard deduction, no exemption, and 10%/12%/22% rates." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied post-TCJA-sunset law (an $8,522 deduction, a $5,435 personal exemption, and 15%/25% brackets) instead of the permanent OBBBA 2026 parameters ($16,100 deduction, no exemption, 10%/12%/22% brackets)." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Used a $16,150 deduction and thresholds of $12,225/$49,725 instead of the 2026 values of $16,100 and $12,400/$50,400. That taxed $1,094 at 22% instead of $468.68." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Applied pre-TCJA law with an $8,372 standard deduction and 15%/25% brackets, leaving $58,597 taxable. Under the OBBBA, 2026 uses a $16,100 deduction and 10%/12%/22% brackets, giving $50,868.68 taxable." -us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"Assumed TCJA expiration with an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets instead of the permanent OBBBA amounts ($16,100 deduction and 10%/12%/22% brackets). It also subtracted the $3,389 ESI premiums from wages." -us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Used the 2023 $13,850 standard deduction instead of 2026's $16,100, giving $6,397.98 on $53,119 taxable. It then submitted an unrelated $4,272 that contradicts its own computation." -us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Used a $16,000 deduction and 2025 thresholds ($11,925/$48,475) instead of $16,100 and $12,400/$50,400, getting $6,127.18. It then submitted $8,252, which matches none of its stated arithmetic." -us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"Reduced tax by a child tax credit/dependent credit for a single filer with no children or dependents, who qualifies for no nonrefundable credits. It also used a wrong taxable income of $49,779 instead of $50,868.68." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It cut wages by the $3,389 ESI premiums and also took the $950 auto loan interest deduction, which brought taxable income down to $46,530. Neither reduction applies; taxable income is $50,868.68. It then submitted $6,224, which does not match its own $5,335.60 figure." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It took a $950 OBBBA auto loan interest deduction, but nothing in the household facts shows a qualifying new vehicle, so the deduction does not apply. Without it, taxable income is $50,868.68, and $468.68 of that falls in the 22% bracket, giving $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It took the $3,389 ESI premiums out of wages and used the 2024 standard deduction of $14,600 instead of 2026's $16,100. It then applied a 22% rate to almost all income, getting $12,761.60, and submitted an unrelated $5,744. The correct calculation uses $50,868.68 of taxable income and the 10%/12%/22% brackets at $12,400 and $50,400." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premiums as pre-tax pay, which made taxable income $47,480 instead of $50,868.68. It also never did the bracket math: even on its own taxable income the tax is $5,449.60, not the $4,700 it guessed." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $16,600 standard deduction instead of the 2026 single amount of $16,100. It then raised its own $5,796 result by an arbitrary $400 to $6,196. With the correct deduction, $468.68 of income falls above $50,400 at 22%, for a total of $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning got the right result: $50,869 of taxable income on the 2026 brackets gives $5,903. It then threw that away and submitted $6,027 as an unexplained 'rounding' adjustment." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly computed about $5,903 from $50,869 of taxable income and the 2026 brackets. It then submitted $6,379, a number its own calculation does not support." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a guessed $15,700 standard deduction instead of the 2026 OBBBA amount of $16,100, and 2025 bracket thresholds ($11,925/$48,475) instead of $12,400/$50,400. Those choices put $2,794 in the 22% bracket instead of $468.68." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It got taxable income right at $50,869, but it estimated the brackets by inflating 2025 thresholds by 2% instead of using the actual 2026 thresholds of $12,400 and $50,400. It also never worked out the bracket amounts and reported a rough $6,154 instead of $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction instead of $16,100, and bracket breakpoints of $12,200/$49,950 instead of 2026's $12,400/$50,400. As a result, $1,369 was taxed at 22% instead of $468.68." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took the $3,389 ESI premiums out of wages, used a $15,400 standard deduction instead of $16,100, and started the 12% bracket at $12,225 instead of $12,400. That left $48,180 of taxable income instead of $50,868.68." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA made the TCJA structure permanent, so 2026 uses a $16,100 standard deduction, no personal exemption, and 10%/12%/22% brackets." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the TCJA-sunset rules: an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA kept the $16,100 deduction and the 10/12/22 brackets for 2026. It also wrongly took the $3,389 ESI premiums out of wages." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of $16,100. Its $3,939 answer is also far below what any 10%/12%/22% calculation gives on about $51,000 of taxable income. The correct tax on $50,868.68 is $1,240 + $4,560 + $103.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and used pre-TCJA personal exemptions, standard deduction, and 15%/25% brackets. Under OBBBA, 2026 keeps the $16,100 standard deduction and the 10/12/22 brackets. It also subtracted the $3,389 ESI premiums, which gross wages do not exclude." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the TCJA-sunset parameters for 2026: an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA made the $16,100 deduction and the 10/12/22 brackets permanent. It also subtracted the $3,389 ESI premiums from wages." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no breakdown, and its $3,737 is far below the correct result. $50,868.68 of taxable income (wages less the $16,100 deduction) on the 2026 brackets gives $1,240 + $4,560 + $103.11. Its answer is consistent with applying a much larger deduction or lower rates than 2026 law provides." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took the $3,389 ESI premiums out of wages as pre-tax pay, which gave $47,480 of taxable income instead of $50,868.68. That dropped all income below the 22% bracket and left out $3,389 that is taxed at 12–22%." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reduced AGI by the $3,389 ESI premiums to $63,580, but AGI equals the full $66,968.68 in wages. Its $5,449.60 is the correct bracket tax on the wrong taxable income of $47,480." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,000 and 2025 brackets ($11,925/$48,475) instead of 2026's $16,100 deduction and $12,400/$50,400 thresholds. That put too much income in the 22% bracket." +us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2024 parameters: a $14,600 standard deduction and brackets at $11,600/$47,150. The 2026 values are $16,100 and $12,400/$50,400. As a result, $5,219 was taxed at 22% instead of $468.68." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It subtracted a 'basic CTC/nonrefundable credit' even though this single filer has no children or dependents, so no CTC or other nonrefundable credit applies. The full $5,903.11 of bracket tax on $50,868.68 of taxable income remains." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It relied on rough approximations of the standard deduction and brackets instead of the actual 2026 values ($16,100 deduction, $12,400/$50,400 thresholds). That produced $5,118, about $785 short of the correct tax on $50,868.68 of taxable income." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction instead of $16,100 and then misapplied the brackets. Its own $50,969 taxable income gives $5,925 on the 2026 brackets, not $5,793. The correct figure is $5,903.11 on $50,868.68." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It got taxable income right at $50,869 but used the wrong bracket breakpoints, overstating the tax by about $82. Only the $468.68 above $50,400 is taxed at 22%, after $1,240 at 10% and $4,560 at 12%." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $3,389 in employer-plan premiums from wages, which gave $47,480 of taxable income. The correct starting point is the full $66,968.68 in wages, leaving $50,868.68 taxable after the $16,100 deduction." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used about $15,000 as the standard deduction (the 2025 amount) instead of the 2026 amount of $16,100, and it never computed the bracket tax. It submitted a rough $5,850 instead of $1,240 + $4,560 + $103.11." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It treated 2026 as a TCJA-sunset year, with an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA keeps the $16,100 deduction, no personal exemption, and the 10/12/22 brackets." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired: it restored a $5,435 personal exemption, used an $8,522 standard deduction, and applied 15%/25% brackets. OBBBA kept the 2026 single standard deduction at $16,100 with 10/12/22 brackets." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $16,150 standard deduction and bracket breakpoints of $12,225/$49,725. The actual 2026 values are $16,100 and $12,400/$50,400, so it taxed $1,094 at 22% instead of $468.68." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA law with an $8,372 standard deduction and 15%/25% brackets. OBBBA made permanent the 2026 $16,100 deduction and the 10/12/22 brackets, which give $50,868.68 of taxable income." +us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired and used an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets, all replaced by OBBBA's permanent $16,100 deduction and 10/12/22 brackets. It also wrongly subtracted the $3,389 ESI premiums from wages." +us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2023 standard deduction of $13,850 instead of 2026's $16,100, which overstated taxable income at $53,119. It then submitted $4,272, which matches neither its own $6,397.98 nor any bracket calculation." +us,scenario_053,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It got taxable income right at $50,869, but it ended the 12% bracket at $50,150 instead of $50,400, giving $4,530 at 12% and $719 at 22%. The correct split is $4,560 at 12% and $468.68 at 22%, for a 22% portion of $103.11." +us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction and 2025 brackets ($11,925/$48,475) instead of 2026's $16,100 and $12,400/$50,400, which produced $6,127.18. It then submitted $8,252, which does not match its own calculation." +us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It used an overstated standard deduction (taxable income of $49,779 instead of $50,868.68). It also subtracted a child or dependent tax credit even though this single filer has no dependents, so no nonrefundable credits apply." us,scenario_053,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a refundable child or other-dependent credit despite the household containing no qualifying child or dependent, and it also treated this high-income childless filer as receiving refundable EITC. Neither eligibility pathway applies, so the claimed $1,966 credit has no qualifying component." us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model applied the correct 6.2% Social Security and 1.45% Medicare rates to a base of $63,580. That base came from subtracting the $3,389 employer-sponsored insurance premium, which treats the premium as a pre-tax Section 125 exclusion the household facts never state. FICA applies to the full $66,969 of gross wages, and 7.65% of that is $5,123.10." us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model explicitly deducted the $3,389 employer-sponsored insurance premium as a pre-tax payroll exclusion ($66,969 - $3,389 = $63,580) before applying 7.65%. The payroll-tax wage base is the full $66,969 of gross wages, so the correct total is $4,152.06 Social Security plus $971.05 Medicare, or $5,123.10." @@ -4545,53 +4595,54 @@ us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model set up us,scenario_053,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $63,580 'after pretax employer-plan premiums'. That subtracts the $3,389 employer-sponsored insurance premium as a cafeteria-plan exclusion the household facts do not state. FICA is levied on the full $66,969 of gross wages, which gives $5,123.10." us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model defined taxable payroll wages as gross wages minus the $3,389 employer-sponsored insurance premium ($63,580), then applied 6.2% and 1.45% and rounded to $4,864. The premium is not excluded from the payroll-tax base. On the full $66,969 of wages, Social Security is $4,152.06 and Medicare is $971.05, for $5,123.10 in total." us,scenario_053,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate instead of Idaho's 2026 5.3% rate. It also taxed an invented $53,000 base built from a made-up '$13,969 exemption', instead of $50,868.68 minus the $4,920 zero bracket ($45,948.68)." -us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"Got the 5.3% rate and $16,100 standard deduction right, but left out the $4,920 single zero bracket. It also lowered AGI to $66,019, which matches subtracting the $950 auto loan interest even though that interest does not reduce Idaho AGI. So it taxed $49,919 instead of $45,948.68." -us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"Used Idaho's repealed six-bracket schedule (1%–7.15%) and a $7,200 standard deduction instead of the flat 5.3% rate above a $4,920 zero bracket with the $16,100 standard deduction. It also subtracted the $3,389 ESI premium from income and invented a $59 personal exemption credit." -us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,state_local_rule,False,"Named the 5.3% flat rate on post-standard-deduction income but never did the calculation; $2,400 is a rough round number. The correct result taxes only $45,948.68, the amount of the $50,868.68 taxable income above the $4,920 zero bracket, giving $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"Used a $15,750 standard deduction instead of the 2026 amount of $16,100 and never applied the $4,920 zero bracket. After computing about $2,715 at 5.3%, it switched to a guessed 4.7% rate to get $2,548." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate and the 2024 $4,673 zero bracket instead of 5.3% above $4,920. It then dropped its own computed $2,631 and reported $2,867 with no basis." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"Its stated method (5.3% on $50,869 above a ~$4,900 zero bracket) gives about $2,436, but it reported $2,650, a figure its own setup doesn't produce. It also cited a grocery-credit offset, but the grocery credit is refundable and does not enter tax before refundable credits." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"Used the 2025 single zero bracket of $4,811 instead of the inflation-indexed 2026 amount of $4,920. That taxed $109 too much at 5.3%, about $5.79 over." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate with no $4,920 zero bracket instead of 5.3% above the zero bracket. It also subtracted the $3,389 ESI premium from AGI and used a $15,000 standard deduction instead of $16,100." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate, a $15,000 standard deduction and no zero bracket. It then threw out its own $2,960 result and reported $3,706 with no derivation; the correct figure is 5.3% × ($50,868.68 − $4,920)." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"Applied 5.3% to the full $50,869 of taxable income and left out Idaho's $4,920 single zero bracket, which cuts the tax by $260.76." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"Used a made-up 5.59% rate, a $15,650 standard deduction and a nonexistent $4,000 Idaho personal exemption. Idaho actually applies the $16,100 standard deduction and taxes income above a $4,920 zero bracket at 5.3%." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"Its $48,180 taxable income matches subtracting the $3,389 ESI premium and a $15,400 standard deduction, when AGI is unreduced and the deduction is $16,100. It also applied 5.3% without the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"Used an $8,300 standard deduction instead of the conforming 2026 single amount of $16,100. It also applied 5.3% to all taxable income, ignoring the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out the $4,920 single zero bracket, overstating the tax by about $261." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate and no zero bracket. Its $49,980 taxable income comes from subtracting the ESI premium and a nonexistent personal exemption instead of just the $16,100 standard deduction from the unreduced $66,968.68 AGI." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"Gave no derivation. $2,652 is roughly 5.3% applied to about $50,000 with no zero bracket, whereas the correct base is $45,948.68: $50,868.68 minus the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"Used a made-up 5.692% rate on about $49,852 with no zero bracket, instead of 5.3% on $45,948.68: the $50,868.68 taxable income minus the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"Used the 2023 5.8% rate instead of 2026's 5.3% and left out the $4,920 zero bracket. It also cut AGI to $63,580 by subtracting the ESI premium, which does not reduce Idaho AGI here." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"Gave no derivation. $3,450 matches an outdated ~5.7–5.8% rate on about $60,000 of taxable income, which understates the $16,100 standard deduction and skips the $4,920 zero bracket in the correct 5.3% calculation." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"Used the old 5.695% rate with no zero bracket. It also subtracted the $3,389 ESI premium to get $63,580 AGI, when Idaho taxable income is $66,968.68 minus $16,100." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Applied the outdated 5.695% rate to about $47,470, which matches subtracting the ESI premium as well as the standard deduction. It never applied the 5.3% rate above the $4,920 zero bracket to the $50,868.68 taxable income." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"Used a $15,000 standard deduction instead of $16,100 and the old 5.695% rate instead of 5.3%. It also left out the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"Invented a '$1,250 per $1,000 of income' Idaho deduction that wipes out all taxable income. Idaho actually subtracts only the $16,100 standard deduction, leaving $50,868.68 taxed at 5.3% above the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"Used a made-up 5.6% rate on the full $50,869 instead of 5.3% on the part above the $4,920 single zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"Gave no rate or base, and $2,050 is too low. The correct calculation taxes $45,948.68 ($66,968.68 − $16,100 − $4,920 zero bracket) at 5.3%, giving $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"Gave no derivation, and $1,195 is less than half the correct tax. That matches a much lower effective rate than Idaho's flat 5.3% on the $45,948.68 above the zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"Got the structure right (5.3% above a zero bracket) but used a stale $15,750 standard deduction instead of $16,100 and an outdated $4,489 zero bracket instead of the 2026 indexed $4,920." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out Idaho's $4,920 single zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"Estimated the 2026 single zero bracket at $4,946 instead of $4,920 (and rounded taxable income to $50,869). That taxed about $26 too little and came in $1.36 low." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out the $4,920 single zero bracket that Idaho taxes at 0%." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"Subtracted a $5,000 health-insurance amount that is not part of Idaho taxable income here, which is just AGI minus the $16,100 standard deduction. It also used the 2025 $4,811 zero bracket instead of 2026's $4,920." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"Applied a progressive bracket schedule to $50,869. Idaho now taxes everything above the $4,920 single zero bracket at a flat 5.3%, so the correct figure is $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"Used a zero bracket of about $4,800 (its base of $46,069 is $50,869 − $4,800) instead of the 2026 indexed $4,920, which taxed about $120 too much at 5.3%." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"Started from $47,480 taxable income by subtracting the $3,389 ESI premium from wages, but AGI is the unreduced $66,968.68. It also used the 2025 $4,811 zero bracket instead of $4,920." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"Used the old 5.8% rate on about $52,000 with no zero bracket, instead of 5.3% on the $45,948.68 of taxable income above $4,920." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"Used an $8,300 standard deduction instead of 2026's $16,100 and the old 5.695% rate instead of 5.3%. It also left out the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"Used an $8,522 standard deduction instead of $16,100 and the outdated 5.695% rate instead of 5.3%. It also left out the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"Got the structure right but overestimated both indexed parameters, using a $16,150 standard deduction instead of $16,100 and a $4,942 zero bracket instead of $4,920. That shrank the taxed base to $45,877 and left it about $4 low." -us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"Applied the repealed 5.8% rate to $58,597, which uses a deduction of only about $8,400 instead of $16,100. It also left out the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out the $4,920 single zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 of taxable income and left out Idaho's $4,920 single zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"Skipped the $16,100 standard deduction and applied the old 5.695% rate to the full $66,969 AGI, getting $3,814. It then reported a different $2,878 with no derivation, and it never used the 5.3% rate above the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"Applied 5.3% to all $50,869 with no $4,920 zero bracket. It then subtracted a made-up $10 'Permanent Building Fund credit'; that fund is a $10 tax, not a credit, and it plays no part in this calculation." -us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"Used the repealed 5.8% rate, subtracted the $3,389 ESI premium from wages, and reached $2,702 by an unexplained adjustment. The correct calculation is 5.3% on the $45,948.68 above the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"Gave no computation, and $1,588 is far too low. Taxing the $50,868.68 of taxable income above the $4,920 zero bracket at 5.3% gives $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Applied the outdated 5.695% rate to a made-up $53,000 base built from a $13,969 'exemption'. It never applied the 5.3% rate above the $4,811 zero bracket and never took Idaho's $5,000 health-insurance premium subtraction. It also omitted the $10 permanent building fund tax. The correct base is $45,868.68 of taxable income." +us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Used a wrong AGI of $66,019 and applied 5.3% to all $49,919 of taxable income, so it skipped the $4,811 zero-rate bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used Idaho's repealed graduated brackets (1%–7.15%) and a $7,200 deduction instead of the $16,100 standard deduction and the 5.3% flat rate above $4,811. It also cut income by the $3,389 ESI premium rather than taking the $5,000 Idaho health-insurance subtraction." +us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Applied about 5.3% to a rough federal-based taxable income with no zero-rate bracket and no Idaho subtraction for the $5,000 of health-insurance premiums. The correct result is 5.3% × ($45,868.68 − $4,811) + $10 building fund tax = $2,186.06." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Computed $2,715 at 5.3% on $51,219, then switched without basis to a 4.7% rate. It used a $15,750 standard deduction instead of $16,100 and never took the $4,811 zero bracket or the $5,000 Idaho health-insurance subtraction. It also omitted the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Used the outdated 5.695% rate, then discarded its own $2,631 calculation for an unsupported $2,867. It never subtracted the $5,000 of health-insurance premiums that Idaho allows and never added the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Started from $50,869 without the $5,000 Idaho health-insurance premium subtraction. It then reached $2,650, which does not follow from its own 5.3%-above-$4,900 method, and wrongly treated the refundable grocery credit as a nonrefundable offset. It also omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Correctly applied 5.3% above the $4,811 zero bracket, but to $50,869. It missed Idaho's $5,000 subtraction for health-insurance premiums, which lowers taxable income to $45,868.68. It also left out the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Subtracted the $3,389 ESI premium from federal AGI, which actually stays at the full $66,968.68 in wages. It used a $15,000 standard deduction and the outdated 5.695% rate with no $4,811 zero bracket. It also counted the $5,000 of premiums only as an itemized medical expense, so it never took the Idaho health-insurance subtraction or added the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Used the 5.695% rate and a $15,000 deduction, then jumped from its own $2,960 to an unsupported $3,706. It stated that health premiums are not deductible for state income tax, but Idaho subtracts the $5,000 of health-insurance premiums. It also skipped the zero bracket and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Applied 5.3% to all $50,869 of taxable income, ignoring the $4,811 zero-rate bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used a made-up 5.59% rate, a $4,000 personal exemption that Idaho does not have, and a $15,650 standard deduction. It omitted the $4,811 zero bracket, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied 5.3% to all of an unsupported $48,180 taxable income, with no $4,811 zero bracket. It did not derive the correct $45,868.68, which comes from subtracting the $5,000 of health-insurance premiums and the $16,100 standard deduction. It also omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Used an $8,300 standard deduction instead of the $16,100 single deduction that Idaho adopts from federal law. It applied 5.3% to all income with no $4,811 zero bracket and missed both the $5,000 health-insurance subtraction and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied 5.3% to all $50,869, ignoring the $4,811 zero-rate bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied the outdated 5.695% rate to $49,980, a figure that includes a nonexistent personal exemption. It omitted the $4,811 zero bracket, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $2,652 matches a flat rate on roughly $50,000 of post-standard-deduction income. It applied neither the $4,811 zero bracket nor the $5,000 Idaho health-insurance subtraction. The correct result is 5.3% × ($45,868.68 − $4,811) + $10 = $2,186.06." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied an outdated 5.692% rate to all of about $49,852. It skipped the 5.3% rate above the $4,811 zero bracket, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Reduced federal AGI by the $3,389 ESI premium, used the repealed 5.8% rate, and added a personal exemption. It never applied 5.3% above the $4,811 zero bracket or took the $5,000 Idaho health-insurance subtraction, and it omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave $3,450 with no derivation. That amount needs an above-5.3% rate on nearly all income with no zero bracket and no $5,000 Idaho health-insurance subtraction. The correct result is 5.3% × ($45,868.68 − $4,811) + $10 = $2,186.06." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied the outdated 5.695% rate to all $47,480, with no $4,811 zero bracket. That base wrongly subtracts the $3,389 ESI premium from AGI instead of Idaho's $5,000 health-insurance premium subtraction. It also omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,703.41 equals 5.695% of about $47,470. That means it used the outdated rate with no zero bracket and reduced AGI by the ESI premium instead of taking the $5,000 Idaho health-insurance subtraction. It also omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used a $15,000 standard deduction and the outdated 5.695% rate on all $51,969. It missed the $16,100 deduction, the 5.3% rate above $4,811, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Invented an Idaho deduction of $1,250 per $1,000 of income, which wiped out all taxable income. Idaho actually allows only the $16,100 standard deduction and the $5,000 health-insurance subtraction. That leaves $45,868.68 taxed at 5.3% above $4,811, plus $10." +us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Applied a made-up 5.6% rate to all $50,869, with no $4,811 zero bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Estimated $2,050 from vague 'state brackets' instead of Idaho's single 5.3% rate above a $4,811 zero bracket. Its taxable income has no stated $5,000 health-insurance subtraction, and it omits the $10 building fund tax that brings the total to $2,186.06." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Gave $1,195 with no derivation, which is about half the correct liability. Taxing $45,868.68 at 5.3% above $4,811 gives $2,176.06, and the $10 building fund tax brings it to $2,186.06." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Correctly used 5.3% above a zero bracket, but with a $15,750 standard deduction instead of $16,100 and a $4,489 threshold instead of $4,811. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Applied 5.3% to all $50,869, ignoring the $4,811 zero-rate bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Used 5.3% above a zero bracket, but started from $50,869 without Idaho's $5,000 health-insurance premium subtraction. It also used $4,946 instead of $4,811 for the zero bracket and omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied 5.3% to all $50,869, with no $4,811 zero bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Got the full income tax calculation right ($2,176.07): the $5,000 health-insurance subtraction, the $16,100 deduction, and 5.3% above $4,811. However, it left out Idaho's $10 permanent building fund tax, which belongs in income tax before refundable credits." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Applied 'progressive' rates to $50,869, but Idaho has one 5.3% rate above a $4,811 zero bracket. It never took the $5,000 health-insurance premium subtraction (taxable income should be $45,868.68) and omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Applied 5.3% above a zero bracket, but started from $50,869 without Idaho's $5,000 health-insurance premium subtraction. It also omitted the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Reduced federal AGI by the $3,389 ESI premium instead of applying Idaho's $5,000 health-insurance premium subtraction, which gave $47,480 instead of $45,868.68 of taxable income. It also omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Used the repealed 5.8% rate on about $52k of income with no zero bracket. It missed the 5.3% rate above $4,811, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used an $8,300 standard deduction instead of $16,100 and the outdated 5.695% rate on all income. It missed the $4,811 zero bracket, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Used an $8,522 standard deduction instead of $16,100 and the outdated 5.695% rate on all income. It missed the $4,811 zero bracket, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Correctly applied 5.3% above a zero bracket, but with a $16,150 deduction and a $4,942 threshold. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Applied the repealed 5.8% rate to $58,597, a base that implies a deduction of only about $8,372 instead of $16,100. It missed the $4,811 zero bracket, the $5,000 Idaho health-insurance subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Applied 5.3% to all $50,869, ignoring the $4,811 zero-rate bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not return a usable value or explanation for state_income_tax_before_refundable_credits. The correct derivation is 5.3% × ($45,868.68 − $4,811) + $10 = $2,186.06." +us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Applied 5.3% to all $50,869, ignoring the $4,811 zero-rate bracket. It also missed Idaho's $5,000 health-insurance premium subtraction and the $10 permanent building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Applied the outdated 5.695% rate to full AGI with no standard deduction, then submitted an unsupported $2,878. It skipped the $16,100 deduction, the $5,000 Idaho health-insurance subtraction, the $4,811 zero bracket, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Applied the outdated 5.695% rate to all $50,869. It skipped the current 5.3% rate above the $4,811 zero bracket, Idaho's $5,000 health-insurance premium subtraction, and the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Treated the $10 permanent building fund as a credit, but it is an additional $10 tax. It also applied 5.3% to all $50,869 with no $4,811 zero bracket and missed Idaho's $5,000 health-insurance premium subtraction." +us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Used the repealed 5.8% rate on AGI reduced by the $3,389 ESI premium, then made unexplained adjustments to reach $2,702. It never applied 5.3% above the $4,811 zero bracket to $45,868.68 (which reflects the $5,000 Idaho health-insurance subtraction) and omitted the $10 building fund tax." +us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Gave $1,588 with no derivation, well below the correct amount. Taxing $45,868.68 at 5.3% above the $4,811 zero bracket gives $2,176.06, and the $10 permanent building fund tax brings it to $2,186.06." us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model said PolicyEngine treats the Idaho grocery credit as nonrefundable and that the $66,969 wages exceed an income limit. In fact the credit is refundable, has no income test, and pays $155 for a full-year resident under 65 in 2026." us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said Idaho has no refundable credits at this income and never considered the grocery credit. That credit is refundable, has no income threshold, and pays $155 per resident under 65 in 2026." us,scenario_053,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 to this full-year resident under 65 in 2026 regardless of income, and answered $0." @@ -4731,51 +4782,52 @@ us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The mode us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed $17,100 × 0.9235 × 0.153 ≈ $2,414, which is close to correct. It then claimed that excluding Additional Medicare Tax and rounding gave $1,596. Additional Medicare Tax is $0 at these earnings, and nothing should be subtracted from regular SECA, so the answer is $2,416.15." us,scenario_054,self_employment_tax,minimax-m3,llm_error,other,False,"The model got net SE earnings of $15,792 and SE tax of about $2,416.18, which is essentially correct. It then submitted $2,419.95, a number that matches none of its own figures. The correct value is $15,791.85 × 0.153 = $2,416.15." us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model correctly reached $2,416 of SE tax. It then applied 0.9235 a second time and subtracted the deductible half of SE tax from the SE tax itself, producing $1,578. The half-SE-tax deduction is an above-the-line deduction against income tax and does not reduce SE tax, which stays at $2,416.15." -us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"It projected a three-person maximum allotment of about $820 instead of the FY2026 value of $785, and used a $211 standard deduction instead of $209. As a result it got $540/month instead of $785 − $280 = $505." -us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"It used the correct $785 maximum and $931 net income, but kept 30% of net income as $279.30. SNAP rounds the expected contribution up to $280, so the allotment is $505/month ($6,060), not $505.70." -us,scenario_054,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a SNAP resource limit to the $14,000 bank balance. North Carolina's broad-based categorical eligibility through TANF non-cash eligibility waives the asset test for this household, which is at 64% of the poverty line. The household is eligible for $505/month." -us,scenario_054,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It zeroed out SNAP based on an asset and income test. Broad-based categorical eligibility removes the asset test, and $1,425/month of gross income passes easily. It also netted SE tax out of income, but SNAP counts the full $17,100 of self-employment earnings." -us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It subtracted half of SE tax from self-employment earnings ($1,324/month instead of $1,425), which SNAP does not do. It also used a $224 standard deduction and a $789 maximum instead of $209 and $785. It then inflated its own $6,456 result to $7,152 with no basis." -us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"Its first pass ($785 maximum minus about $277 = $508/month) came close to the correct $505. It then dropped that result for an unsupported $707/month 'refined estimate', which matches no SNAP calculation." -us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"It assumed an $800 maximum allotment and came up with $583/month without showing the work. With net income of $931, the contribution is $280 and the benefit is $785 − $280 = $505/month." -us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"Its inputs were all correct ($1,425 gross, $285 earned income deduction, $209 standard deduction, $785 maximum), but it did not round 30% of net income ($279.30) up to $280. That gave $505.70/month instead of $505." -us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a 40% self-employment cost deduction that has no basis here. No business expenses are listed, so SNAP counts the full $1,425/month. It also used a $228 standard deduction and a $766 maximum, and submitted $5,532 even though its own work concluded $7,548." -us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"Even on its own figures (a $768 maximum minus about $280), the benefit comes to about $488/month, but it stated about $350/month. It also double-counted the 20% earned income deduction and used a $204 standard deduction instead of $209." -us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"It computed $785 − 0.30 × $931 = $505.70 without rounding the expected contribution up to $280. The correct benefit is $505/month, or $6,060 a year." -us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,416 of SE tax from self-employment earnings before computing SNAP income. SNAP counts the full $1,425/month. It also used an outdated $198 standard deduction and a $775 maximum instead of $209 and $785." -us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,It used outdated parameters: a $781 maximum allotment and a $202 standard deduction instead of the FY2026 values of $785 and $209. That produced net income of $938 and $499/month instead of $931 net and $505/month. -us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $784 maximum and a $208 standard deduction instead of $785 and $209. It also kept the contribution at $279.60 rather than rounding it up to a whole dollar, which gave $504.40/month instead of $505." -us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It overstated the three-person maximum allotment as $787 instead of $785. It also left the contribution at $279.30 instead of rounding it up to $280, which gave $507.70/month instead of $505." -us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,It used an old $766 maximum and a $198 standard deduction (net income $942) instead of the FY2026 values of $785 and $209 (net income $931). The result was $483.40/month instead of $505. -us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $3,720 answer implies $310/month, which badly understates the benefit. The correct figure is the $785 maximum minus 30% of $931 net income, rounded up to $280, which equals $505/month." -us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $5,800.80 matches the outdated $766 maximum and $198 standard deduction ($766 − 0.3 × $942 = $483.40). The FY2026 values are $785 and $209, which give $505/month." -us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,It used a stale $747 three-person maximum allotment and a $198 standard deduction instead of $785 and $209. That gave $464/month instead of $505. -us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It claimed income exceeds the SNAP limits, but $1,425/month of gross income is 64% of the $2,220.83 monthly poverty guideline. That is far below both the gross and net income tests, so the household gets $505/month." -us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $7,548 ($629/month) matches cutting self-employment income by a 40% cost allowance to $855/month and using a $766 maximum. SNAP counts the full $1,425/month with no listed expenses, so the benefit is $505/month." -us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $485/month matches an outdated $766 maximum minus a contribution of about $281. The FY2026 parameters give a $785 maximum minus a $280 contribution, which is $505/month." -us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $487/month matches the FY2025 parameters: a $768 maximum and a $204 standard deduction, giving net income of $936 and a $281 contribution. The FY2026 values are $785 and $209, which give $505/month." -us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $228/month standard deduction and an $800/month ($9,600/year) maximum allotment instead of $209 and $785. That produced $6,317 instead of 12 × ($785 − $280) = $6,060." -us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,It computed net income of $931 correctly but overstated the three-person maximum allotment as $816 instead of $785. That gave $537/month instead of $505. -us,scenario_054,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated countable income as 'very low' and awarded about $739/month. The household has $931 of net income after deductions, which creates a $280 expected contribution against the $785 maximum, for $505/month." -us,scenario_054,snap,gpt-5.4-nano,llm_error,missing_output,False,"It defaulted SNAP to 0 without running the benefit calculation, even though it had household size, self-employment income, and state. The standard calculation gives $505/month." -us,scenario_054,snap,gpt-5.5,llm_error,other,False,"It set up the right formula ($785 − 30% × $931) but got $575 by mistake. The correct arithmetic, with the $279.30 contribution rounded up to $280, gives $505/month." -us,scenario_054,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It applied a resource limit to the $14,000 bank balance. North Carolina's broad-based categorical eligibility through TANF non-cash eligibility waives the SNAP asset test for this low-income household." -us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"It got net income of $931 and correctly noted that broad-based categorical eligibility removes the asset test. It then rounded the $505.70 benefit to $506 instead of rounding the $279.30 contribution up to $280, which gives $505/month." -us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,It rounded $785 − $279.30 to $506 instead of rounding the 30% contribution up to $280 first. The correct monthly allotment is $505. -us,scenario_054,snap,gpt-6-astra,llm_error,other,False,"Its formula 12 × [785 − 0.30 × (1425 × 0.80 − 209)] leaves the contribution at $279.30. SNAP rounds it up to $280, which gives $505/month and $6,060 a year." -us,scenario_054,snap,gpt-6-luna,llm_error,other,False,It used $505.70/month because it did not round 30% of the $931 net income up to $280. The correct allotment is $505/month. -us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"It claimed both income and assets exceed the limits. Gross income is 64% of the poverty line, and broad-based categorical eligibility waives the asset test, so the household gets $505/month." -us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"It used an outdated $766/month maximum ($9,192/year) and a $198/month standard deduction ($2,376/year) instead of the FY2026 values of $785 and $209. That gave $5,801 instead of $6,060." -us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"It used the FY2025 parameters: a $768 maximum and a $204 standard deduction ($2,448/year). The FY2026 values are $785 and $209, which give a $280 contribution and $505/month." -us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,"It used the FY2025 $768 maximum and $204 standard deduction instead of the FY2026 values of $785 and $209. It also did not round the contribution up to whole dollars, which gave $5,846.40 instead of $6,060." -us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It overstated the three-person maximum allotment as about $811 instead of $785, and used a standard deduction of about $205 instead of $209. That gave $531/month instead of $505." -us,scenario_054,snap,inkling,llm_error,categorical_eligibility,False,"It ruled the household ineligible because of a resource limit on the $14,000 bank balance. North Carolina's broad-based categorical eligibility through TANF non-cash eligibility removes the asset test." -us,scenario_054,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied a countable resource limit for households without elderly or disabled members. That limit does not apply here, because North Carolina's broad-based categorical eligibility waives the asset test for a household at 64% of the poverty line." -us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"It handled net income ($931), the rounded-up $280 contribution, and the asset waiver correctly, but used a $783 maximum allotment instead of the FY2026 three-person value of $785. That gave $503/month instead of $505." -us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"It got the comparison backwards. $17,100 of annual income is below its own roughly $26,000 limit at 130% of the poverty line, so the household passes the gross income test and gets $505/month." -us,scenario_054,snap,ox-alpha,llm_error,other,False,It used all the correct FY2026 inputs but kept the expected contribution at $279.30 instead of rounding it up to $280. That gave $505.70/month instead of $505. -us,scenario_054,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It awarded a $934 'maximum' benefit with no income reduction. The three-person maximum is $785, and the household's $931 of net income creates a $280 contribution, which leaves $505/month." -us,scenario_054,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used $1,579 of monthly earnings instead of $1,425 and claimed net income exceeds the limit. In fact, $931 of net income is 42% of the poverty line, so the household gets $505/month." +us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model projected a maximum allotment of ~$820 for three people from the wrong FY2025 base ($768). The actual FY2026 figure is $785, so the correct benefit is $785 − $280 = $505/month, not $540." +us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"The model had the right inputs ($785 maximum, $931 net income) but subtracted the unrounded 30% contribution of $279.30. SNAP rounds the contribution up to $280, which gives $505/month and $6,060/year, not $505.70 × 12." +us,scenario_054,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated the $14,000 bank balance as disqualifying. NC's broad-based categorical eligibility (TANF non-cash eligibility) waives the SNAP resource limit, so the household is eligible for $505/month." +us,scenario_054,snap,claude-haiku-5.5,llm_error,asset_resource,False,"The model zeroed out SNAP on asset and income grounds. BBCE waives the asset test in NC, and gross income of $1,425/month is only 64% of the poverty guideline, so the household gets $505/month." +us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted half of SE tax from self-employment income, but SNAP counts the full $1,425/month when no business expenses are listed. It also used a $224 standard deduction and a $789 maximum instead of $209 and $785, then inflated the result to $7,152 with no basis." +us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"The model's own computation reached about $508/month, close to the correct $785 − $280 = $505. It then dropped that result for an unsupported 'refined' $707/month." +us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,The model used an $800 maximum allotment and guessed ~$583/month without doing the subtraction. The $785 maximum minus the $280 contribution on $931 net income gives $505/month. +us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"The model subtracted 30% of $931 as $279.30 without rounding it up to the whole dollar. With the contribution rounded up to $280, the allotment is $505/month, or $6,060/year, not $6,068.40." +us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a nonexistent 40% cost-of-doing-business cut to self-employment income, giving $855/month instead of the full $1,425. It also used outdated $228 and $766 parameters and submitted $5,532, which contradicts its own $7,548 calculation." +us,scenario_054,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used the FY2025 $768 maximum and a ~$204 standard deduction instead of $785 and $209. It then miscalculated: even its own figures give about $488/month, not $350. The correct benefit is $505/month." +us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"The model used the 30% contribution unrounded at $279.30 instead of rounding up to $280. That turned $505/month ($6,060/year) into $505.70 and $6,068." +us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model deducted $2,416 of SE tax from self-employment income, but SNAP gross earned income is the full $1,425/month. It also used a $198 standard deduction and a $775 maximum instead of $209 and $785." +us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used a $781 maximum allotment and a $202 standard deduction instead of the FY2026 values of $785 and $209. That raised net income to $938 and cut the benefit to $499 instead of $505. +us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $784 maximum and a $208 standard deduction instead of $785 and $209. It also left the 30% contribution unrounded ($279.60) rather than rounding up to the next whole dollar, which gave $504.40 instead of $505." +us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $787 maximum allotment instead of $785. It also left the contribution at $279.30 instead of rounding up to $280, which gave $507.70/month instead of $505." +us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model used outdated figures: a $766 maximum allotment and a $198 standard deduction (net $942) instead of the FY2026 $785 and $209 (net $931). That gave $483.40/month instead of $505. +us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model gave no derivation, and its $310/month is far below the correct $505. The correct figure is $785 minus a $280 contribution on $931 net income ($1,425 less $285 earned-income and $209 standard deductions)." +us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $483.40/month matches a $766 maximum minus 30% of $942 net income, i.e. outdated maximum allotment and $198 standard deduction values. The FY2026 parameters are $785 and $209, which give $505/month." +us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a $747 maximum allotment and a $198 standard deduction instead of the FY2026 $785 and $209. That gave $464/month instead of $505. +us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model claimed income exceeds SNAP limits. Gross income of $1,425/month is 64% of the $2,220.83 poverty guideline and net income of $931 is 42%, so the household passes both tests and gets $505/month." +us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"$7,548 ($629/month) matches cutting self-employment income by 40% to $855/month and using outdated $766 and $228 parameters. SNAP counts the full $1,425/month, which gives net income of $931 and $505/month." +us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"$485/month matches the FY2025 $768 maximum minus 30% of $942 net income, which uses the $198 FY2025 standard deduction. The FY2026 $785 maximum and $209 deduction give $505/month." +us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,$487/month matches the FY2025 $768 maximum allotment instead of the FY2026 $785. The correct benefit is $785 − $280 = $505/month. +us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used an $800/month maximum ($9,600/year) and a $228/month standard deduction instead of $785 and $209. It also did the calculation on annual totals rather than with the monthly rounded contribution of $280." +us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,"Net income ($931) was right, but the model used an $816 maximum allotment instead of the FY2026 $785. The correct benefit is $785 − $280 = $505/month." +us,scenario_054,snap,gpt-5.4-mini,llm_error,other,False,"The model treated countable income as 'very low' and gave a near-maximum $739/month. It never subtracted the $280 expected contribution on $931 net income from the $785 maximum, which gives $505/month." +us,scenario_054,snap,gpt-5.4-nano,llm_error,other,False,"The model said eligibility couldn't be assessed and defaulted to $0, ignoring the given household size of three and $1,425/month of self-employment income. Those facts produce a $505/month allotment under BBCE." +us,scenario_054,snap,gpt-5.5,llm_error,other,False,The model had the correct $785 maximum and $931 net income but got the subtraction wrong: it reported $575/month when $785 − $280 is $505. +us,scenario_054,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model applied an NC resource limit to the $14,000 bank balance. NC confers categorical eligibility through TANF non-cash eligibility, which removes the asset test, so the household gets $505/month." +us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"The model rounded the $505.70 benefit up to $506 instead of rounding the 30% contribution ($279.30) up to $280. The correct allotment is $505/month, or $6,060/year." +us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,The model rounded the benefit to $506 instead of rounding the expected contribution up to $280 first. The correct allotment is $785 − $280 = $505/month. +us,scenario_054,snap,gpt-6-astra,llm_error,other,False,"The model's formula used 0.30 × $931 = $279.30 unrounded. SNAP rounds the contribution up to $280, which gives $505/month and $6,060/year." +us,scenario_054,snap,gpt-6-luna,llm_error,other,False,"The model annualized $505.70 instead of rounding the $279.30 contribution up to $280. The correct allotment is $505/month, or $6,060/year." +us,scenario_054,snap,grok-4.3,llm_error,asset_resource,False,"The model claimed income and assets exceed the limits. BBCE waives the asset test, and $1,425/month gross is 64% of the poverty guideline, so the household gets $505/month." +us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used a $766/month maximum ($9,192/year) and a $198/month standard deduction ($2,376/year) instead of the FY2026 $785 and $209. That understated the benefit relative to $505/month." +us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,The model used the FY2025 $768 maximum and a $204 standard deduction instead of the FY2026 $785 and $209. That gave about $487/month instead of $505. +us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,The model used the FY2025 $768 maximum and a $204 standard deduction instead of the FY2026 $785 and $209. It also skipped the monthly round-up of the 30% contribution. +us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,The model used an $811 maximum allotment and a ~$205 standard deduction instead of $785 and $209. That gave $531/month instead of $505. +us,scenario_054,snap,inkling,llm_error,asset_resource,False,"The model treated $14,000 in bank assets as exceeding an NC resource limit. NC's broad-based categorical eligibility (TANF non-cash eligibility) eliminates the asset test, so the household gets $505/month." +us,scenario_054,snap,kimi-k2.6,llm_error,asset_resource,False,"The model applied a countable-resource limit, but NC confers categorical eligibility through TANF non-cash eligibility, which waives the SNAP asset test. The household gets $785 − $280 = $505/month." +us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly rounded the contribution up to $280, but used a $783 maximum allotment instead of the FY2026 $785. That gave $503/month instead of $505." +us,scenario_054,snap,minimax-m3,llm_error,other,False,"The model said $17,100 exceeds 130% of the poverty line (~$26,000), which is backwards: the income is well below that limit (64% of the guideline). The household passes and gets $505/month." +us,scenario_054,snap,muse-wonder,llm_error,thresholds_rates,False,The model used the FY2025 $768 maximum and a smaller standard deduction (net $936) instead of the FY2026 $785 and $209 (net $931). That gave $487/month instead of $505. +us,scenario_054,snap,ox-alpha,llm_error,other,False,"The model subtracted $279.30 rather than rounding the 30% contribution up to $280. The correct allotment is $505/month and $6,060/year, not $505.70 and $6,068.40." +us,scenario_054,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model awarded a full maximum benefit of $934/month, which overstates the $785 FY2026 maximum for three. It also skipped the $280 expected contribution on $931 net income, which brings the benefit to $505/month." +us,scenario_054,snap,qwen3.8-max,llm_error,other,False,"The model used $1,579 of self-employment earnings and claimed net income exceeds the limit. Net income is $931, 42% of the $2,220.83 poverty guideline, which passes the net income test and gives $505/month." us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly applied North Carolina's $19,125 head-of-household standard deduction against federal AGI of $15,892 and stated twice in its own reasoning that NC taxable income floors at $0 and ""NC tax = 0,"" then discarded that conclusion and submitted $245. No NC computation produces $245 from these facts: at the 2026 flat rate of 3.99% it implies $6,140 of taxable income, which the household does not have once the standard deduction is applied." us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"After correctly computing $0 NC taxable income using the head-of-household standard deduction of $19,125, the model reassigned the filer to single status and its $12,750 standard deduction despite two qualifying children in the household, yielding $3,142 of taxable income and $133.50 of tax. It then submitted $596, a figure tied to neither derivation and roughly 3.75% of gross self-employment income with no standard deduction at all." us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied the $12,750 single-filer standard deduction rather than the $19,125 head-of-household deduction the head qualifies for with two dependent children, and used the 2025 rate rather than the 2026 flat rate of 3.99%: $15,892 − $12,750 = $3,142 taxed at 4.5% is $141.39, matching its $142. Under the correct head-of-household deduction the base is zero and the tax is $0." @@ -4845,105 +4897,107 @@ us,scenario_056,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"T us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, violating the required output contract." us,scenario_056,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model reversed the $400 threshold rule: $4,340 of self-employment income is above, not below, $400. Applying the 92.35% net-earnings adjustment and 15.3% combined rate yields $613.22." us,scenario_056,self_employment_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly declared $4,340 below the $400 self-employment-tax threshold. Because the threshold is exceeded, the 92.35% net-earnings base is taxed at 15.3%, producing $613.22." -us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added a standard utility allowance even though no utility expenses were listed, and counted the $564/month mortgage interest as a shelter cost, so it hit the $744 excess-shelter cap and got a $159/month benefit. Shelter costs are only the $995.75/month property taxes with no SUA, so the deduction is $395.49, net income is $805, the formula gives $63, and NJ's $95 minimum sets the benefit at $1,140 a year." -us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Treated the $6,773 home mortgage interest plus property tax ($18,722) as SNAP shelter costs and took the capped $8,928 excess-shelter deduction. Under the reference reading, mortgage interest is a tax-only input and shelter costs are just the $11,949 property tax. That gives a $395.49/month deduction, $805 net income and a $63 formula benefit, raised to NJ's $95/month minimum." -us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Claimed $17,839 of gross income exceeds the one-person SNAP limit, but $1,486.54/month is only 1.12 times FPL, well under NJ's broad-based categorical eligibility gross limit. It also treated a $100,341 mortgage balance, which is a debt and not an asset, as disqualifying resources. The household is eligible, and NJ's $95 minimum gives $1,140." -us,scenario_056,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no income or deduction computation. The household is eligible with $805 net income after the standard, earned-income and property-tax shelter deductions. The $63 formula benefit is raised to NJ's $95/month minimum, which is $1,140 a year." -us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for the $995.75/month property taxes and overstated net income at about $1,400/month, so it wrongly failed the net income test. Net income is $805 (61% FPL). An eligible one-person household gets at least NJ's $95/month minimum even when 30% of net income exceeds the maximum allotment." -us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit by assuming high assets from home ownership and the financial assistance. The home is an excluded resource, no countable assets are listed, and NJ's broad-based categorical eligibility removes the asset test. The household is eligible and receives NJ's $95/month minimum because the formula gives only $63." -us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Claimed $17,839 of income exceeds the gross and net limits. Gross income is 1.12 times FPL, under NJ's broad-based categorical eligibility limit, and net income after the property-tax shelter deduction is 0.61 times FPL. The $63 formula benefit is raised to NJ's $95/month minimum." -us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 financial assistance and used only $487/month of income. SNAP counts it as unearned income, so gross income is $1,486.54/month. With the property-tax-only shelter deduction of $395.49, net income is $805, not $0, so the benefit is the $95 NJ minimum rather than the maximum allotment." -us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Correctly found the household passes the net income test, then set the benefit to $0 because 30% of net income exceeded the maximum allotment. An eligible one-person household always gets the minimum benefit, which is $95/month in New Jersey, so the answer is $1,140. It also left out the $395.49 property-tax excess shelter deduction." -us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Claimed countable income exceeds 130% of FPL. Gross income of $1,486.54/month is 1.12 times the $1,330 guideline and passes, and NJ uses broad-based categorical eligibility anyway. Net income is $805, the formula gives $63, and NJ's $95 minimum applies." -us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $12,000 financial assistance, which SNAP counts as unearned income, and counted mortgage interest so shelter costs hit the cap. Correct net income is $805/month with only the $395.49 property-tax shelter deduction, so the formula gives $63 and NJ's $95 minimum applies, not the $298 maximum." -us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Set the benefit to $0 because 30% of net income reached the maximum allotment. It left out the $395.49 property-tax excess shelter deduction and ignored that an eligible one-person household receives NJ's $95/month minimum benefit, which is $1,140 a year." -us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the ABAWD three-month time limit and cut eligibility to zero months, but the household passes the SNAP work requirement test under PolicyEngine's rules, so no time limit applies. It is eligible with $805 net income and receives NJ's $95/month minimum." -us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Counted the home mortgage interest as a shelter cost and took a capped $712 excess-shelter deduction, giving net income of $498 and $142/month. Under the reference reading, only the $995.75/month property tax is a shelter cost. The deduction is then $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Concluded that 30% of net income exceeds the maximum allotment and gave $0. It left out the $395.49 property-tax excess shelter deduction, which brings net income to $805 and the formula to $63, and it ignored NJ's $95/month minimum benefit for eligible households." -us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Built a capped $672 shelter deduction from mortgage interest, property taxes and utility expenses, but no utility expenses were listed. Shelter costs are only the $995.75/month property taxes, so the deduction is $395.49 and net income is $805, not $544. The $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds the SNAP threshold. $1,486.54/month is 1.12 times FPL, well under NJ's broad-based categorical eligibility gross limit. The household is eligible, and the $63 formula benefit is raised to NJ's $95/month minimum." -us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Applied the maximum excess shelter deduction using mortgage interest plus property tax, which gives $489 net income and $158/month. Under the reference reading, mortgage interest is a tax-only input, so the shelter deduction is the uncapped $395.49 from property tax alone. Net income is $805 and the formula gives $63, raised to NJ's $95 minimum." -us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted only self-employment and alimony income, dropped the $1,000/month financial assistance, and awarded the full $292 maximum. With the assistance counted as unearned income, net income is $805/month, and the benefit is NJ's $95 minimum." -us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no supporting computation. The household is eligible with $805 net income after the standard, earned-income and property-tax shelter deductions. The $63 formula benefit is raised to NJ's $95/month minimum, which is $1,140 a year." -us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Its $140.72/month benefit implies net income of about $500, which requires a capped excess-shelter deduction built from mortgage interest plus property tax. Under the reference reading, only the $995.75/month property tax counts, giving a $395.49 deduction and $805 net income. The $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Awarded the full $292 maximum allotment, which implies zero net income. That is only possible if the $12,000 financial assistance is ignored and an outsized shelter deduction is taken. Correct net income is $805/month, so the benefit is NJ's $95 minimum." -us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Disqualified the head as a non-working ABAWD, but the household passes the SNAP work requirement test under PolicyEngine's rules. It is eligible with $805 net income and receives NJ's $95/month minimum." -us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564.08/month of mortgage interest to the $995.75 property tax and capped the excess shelter deduction at $733, giving $159.63/month. Under the reference reading, mortgage interest is a tax-only input, so the excess shelter deduction is $995.75 − $600.27 = $395.49. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 financial assistance as TANF-type welfare and left out the property-tax shelter deduction. SNAP counts the assistance as unearned income, so gross income is $1,486.54 and net income is $805. The formula gives $63, which NJ's $95 minimum replaces." -us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 without computing income or deductions. The household passes the gross and net income tests ($805 net, 61% FPL), and the $63 formula benefit is raised to NJ's $95/month minimum, which is $1,140 a year." -us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 by citing the alimony and self-employment income without running the benefit formula. Net income after the standard, earned-income and property-tax shelter deductions is $805, the formula gives $63, and NJ's $95 minimum applies." -us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly found the household eligible and the formula benefit below the minimum, but applied the federal $23 minimum for one-person households. New Jersey funds a $95/month state minimum benefit, so the answer is $1,140, not $276." -us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Set the allotment to $0 because net income was too high. It left out the $395.49 property-tax excess shelter deduction, which brings net income to $805 and the formula to $63, and it ignored NJ's $95/month minimum for eligible households." -us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess-shelter deduction by counting mortgage interest with property tax, giving $159/month. Under the reference reading, mortgage interest is a tax-only input and the shelter deduction is the uncapped $395.49 from property tax alone. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Built a capped shelter deduction from the listed mortgage interest and property taxes, giving $158/month. Under the reference reading, only the $995.75/month property tax is a shelter cost, so the deduction is $395.49 and net income is $805. The $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Took the capped $8,928 annual shelter deduction by counting mortgage interest with property tax. Under the reference reading, only the $11,949 property tax counts, so the excess shelter deduction is $395.49/month and annual net income is about $9,660. The formula gives $63/month, which NJ's $95 minimum replaces." -us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,Gave $0 because 30% of net income exceeded the maximum allotment. It left out the $395.49 property-tax excess shelter deduction and ignored that an eligible one-person household receives NJ's $95/month minimum benefit. -us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Explicitly counted mortgage interest as a shelter cost and applied the $8,928 cap. Under the reference reading, mortgage interest is a tax-only input and the shelter deduction is the uncapped $395.49/month from property tax alone. That gives a $63 formula benefit, raised to NJ's $95 minimum." -us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Declared the household ineligible on income and assets. Gross income is 1.12 times FPL under NJ's broad-based categorical eligibility limit, no countable assets are listed, and the home is excluded. The household is eligible and receives NJ's $95/month minimum." -us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Admitted income is within limits but still gave $0. The $995.75/month property tax is a countable shelter cost producing a $395.49 deduction, and an eligible one-person household receives at least NJ's $95/month minimum, which is $1,140." -us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Treated the absence of rent and utilities as zero shelter costs, but the $11,949 real estate tax is a homeowner shelter cost. That gives a $395.49 excess shelter deduction and $805 net income. The $63 formula benefit is raised to NJ's $95/month minimum rather than $0." -us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Counted only alimony and self-employment income, dropping the $12,000 financial assistance that SNAP counts as unearned income. It also set shelter costs to $0 despite the property taxes. Correct net income is $805/month, so the benefit is NJ's $95 minimum, not $229/month." -us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Left out the property-tax excess shelter deduction and returned $0 once 30% of net income exceeded the maximum allotment. Even its own $1,123/month net income is under the net limit, so the household is eligible and receives NJ's $95/month minimum." -us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Included mortgage interest in shelter costs and applied the $712 cap, giving $493 net income and $144/month. Under the reference reading, only the $995.75/month property tax is a shelter cost, so the deduction is $395.49 and net income is $805. The $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value or explanation, so there was no answer to score." -us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Summed mortgage interest and real estate taxes into $1,560.17/month of shelter costs and capped the excess at $744. Under the reference reading, mortgage interest is a tax-only input, so shelter costs are $995.75 and the uncapped deduction is $395.49. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed income exceeds the one-person limits. Gross income is 1.12 times FPL, under NJ's broad-based categorical eligibility limit, and net income is 0.61 times FPL. The household is eligible and receives NJ's $95/month minimum." -us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess shelter deduction using $564/month of mortgage interest plus $996 of property tax, giving net income of about $500. Under the reference reading, mortgage interest is a tax-only input and the deduction is the uncapped $395.49 from property tax. Net income is $805 and the $63 formula benefit is raised to NJ's $95 minimum." -us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Contradicted itself on the gross limit and then asserted that income exceeds the threshold. Gross income of $1,486.54/month passes, and after the property-tax shelter deduction net income is $805. The $63 formula benefit is raised to NJ's $95/month minimum." -us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds 130% of FPL and net income exceeds the net limit. Gross income is 1.12 times FPL, and net income after the $395.49 property-tax excess shelter deduction is 0.61 times FPL. The household is eligible and receives NJ's $95/month minimum." +us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added the $564/mo of mortgage interest (plus an SUA, though no utilities were listed) to shelter costs and took the capped ~$744 excess shelter deduction, which gave net income of ~$462 and $159/mo. PolicyEngine's shelter cost is only the $995.75/mo of property taxes, so the excess shelter deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95/mo state minimum ($1,140/yr)." +us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Treated mortgage interest plus property tax ($18,722/yr) as shelter costs and took the full $8,928 capped shelter deduction, which gave annual net income of $5,535. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Claimed $17,839 of gross income exceeds the SNAP limit and treated the $100,341 mortgage balance as an asset. Gross income is only 112% of FPL, well under New Jersey's BBCE gross limit, and a mortgage balance is a debt, not a countable resource. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Gave no computation and returned $0. The household passes the gross test (112% of FPL) and the net test (61% of FPL). Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Applied no excess shelter deduction even though the household pays $995.75/mo in real estate taxes, then wrongly said net income of ~$1,400 exceeds the net limit. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit because of assets it assumed from home ownership and the financial assistance. A home is an excluded resource, and New Jersey's BBCE removes the asset test. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Asserted that $17,839 of income exceeds the gross and net limits for one person. Gross income of $1,486.54/mo is 112% of FPL and net income of $805 is 61% of FPL, so both tests pass. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Left the $12,000 of financial assistance ($1,000/mo) out of income, used only $487/mo, and also counted mortgage interest as a shelter cost, which pushed net income to $0 and gave the maximum allotment. PolicyEngine counts the financial assistance as unearned income, giving net income of $805 and a $63 formula benefit that New Jersey's $95 minimum raises to $1,140/yr." +us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Found the household eligible but returned $0 because 30% of net income exceeded the maximum allotment. It left out the $395.49 excess shelter deduction from the $995.75/mo of property taxes and did not apply New Jersey's $95/mo state minimum benefit, which applies to eligible households whose formula benefit is below $95." +us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Claimed countable income exceeds 130% of FPL. Gross income of $1,486.54/mo is 112% of the $1,330 guideline, and net income of $805 is 61% of it. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly left the $12,000 of financial assistance out of income and added mortgage interest to shelter costs, which drove net income to zero and gave the full $298/mo allotment. PolicyEngine counts the assistance as unearned income ($1,000/mo) and only property taxes as shelter costs, so net income is $805, the formula benefit is $63/mo, and NJ's $95 minimum sets the annual benefit at $1,140." +us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Stopped at a non-positive formula benefit after only the earned-income and standard deductions. It omitted the property-tax shelter deduction and NJ minimum used in the frozen calculation. The frozen $1,140 annual value assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." +us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Disqualified the household under the ABAWD time limit. The frozen reference passes the work test because it assumes 40 weekly work hours. The prompt leaves hours unlisted and defaults them to zero; the developer decision recognizes that reading and excludes the output, so the ABAWD conclusion is not disproved by the $1,140 reference." +us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Took a capped $712 excess shelter deduction by counting mortgage interest with the property taxes, which gave net income of $498 and $142/mo. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Concluded that 30% of net income exceeds the maximum allotment and returned $0. Its $0 explanation says 30% of net income exceeds the maximum allotment, without showing deductions or discussing the NJ minimum used in the frozen calculation. The frozen $1,140 annual value assumes 40 weekly work hours and property-tax-only shelter costs; work hours and occupied-home mortgage status are unlisted, so the output remains excluded." +us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Took a capped $672 shelter deduction based on mortgage interest, property tax, and utilities that were never listed, which gave net income of $544. PolicyEngine's shelter cost is only the $995.75/mo of property taxes, giving a $395.49 deduction, $805 net income, and a $63 formula benefit that New Jersey's $95 minimum raises to $1,140/yr." +us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds SNAP limits. Gross income of $1,486.54/mo is 112% of FPL, below the income limit. The frozen $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." +us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Counted mortgage interest with property taxes to reach the maximum excess shelter deduction, which gave net income of $489 and $158/mo. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Listed self-employment and alimony as countable income and returned an estimated maximum of $292/mo without explaining its treatment of the $12,000 financial assistance. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted assumptions leave the output excluded from scoring." +us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave no derivation for $0. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Its $140.72/mo matches a capped shelter deduction built from mortgage interest plus property taxes. PolicyEngine's shelter cost is only the $995.75/mo of property taxes, giving a $395.49 deduction, $805 net income, and a $63 formula benefit that New Jersey's $95 minimum raises to $1,140/yr." +us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Returned $3,504/yr, equal to $292 × 12, without a detailed income or shelter calculation. The release convention holds the FY2026 one-person maximum at $298/mo. Mortgage payments including interest count as SNAP shelter costs if the household occupies the mortgaged home; the frozen $1,140 reference instead assumes 40 weekly work hours and property-tax-only shelter costs, and remains excluded because work hours and occupied-home mortgage status are unlisted." +us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Disqualified the head as an ABAWD with zero work hours. The frozen reference passes the work test because it assumes 40 weekly work hours. The prompt leaves hours unlisted and defaults them to zero; the developer decision recognizes that reading and excludes the output, so the ABAWD conclusion is not disproved by the $1,140 reference." +us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564/mo of mortgage interest to the $995.75/mo of property taxes and capped the shelter deduction at $733, which gave $160/mo. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 of financial assistance as TANF-type welfare and applied no shelter deduction, so it computed from $487/mo of income. PolicyEngine counts the assistance as unearned income, giving net income of $805 after the $395.49 property-tax shelter deduction and a $63 formula benefit that New Jersey's $95 minimum raises to $1,140/yr." +us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 without computing eligibility. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Said there were no low-income indicators and returned $0. Monthly gross income of $1,486.54 is 112% of FPL, but the frozen $1,140 annual benefit also assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." +us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Applied about $23/mo as the federal minimum; c_snap_hold_fy2026 holds the federal minimum at $24/mo, while New Jersey's state minimum is $95/mo. The shown frozen formula benefit of $63.18 exceeds the federal minimum. The $1,140 annual reference remains excluded because work hours and occupied-home mortgage status are unlisted." +us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Returned a zero allotment because net income was too high. Its $0 explanation attributes the result to high net income without showing deductions or discussing the NJ minimum used in the frozen calculation. The frozen $1,140 annual value assumes 40 weekly work hours and property-tax-only shelter costs; work hours and occupied-home mortgage status are unlisted, so the output remains excluded." +us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess shelter deduction by counting mortgage interest with property taxes, which gave $159/mo. Mortgage interest counts if the mortgage is on the occupied home; the sidecar recheck reproduces $1,908/yr with 40 weekly work hours and that shelter-cost reading. The frozen $1,140 reference uses different unlisted assumptions and remains excluded." +us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Built a capped shelter deduction from mortgage interest plus property taxes, which gave $158/mo. PolicyEngine's shelter cost is only the property taxes ($995.75/mo), so the excess deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95/mo minimum." +us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Took the full $8,928 capped shelter deduction by counting mortgage interest as a shelter cost, which gave annual net income of $5,535. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,Returned $0 because 30% of net income exceeded the maximum allotment. It left out the $395.49 property-tax shelter deduction and did not apply New Jersey's $95/mo minimum benefit for eligible households. +us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Counted mortgage interest as a shelter cost and took the full $8,928 capped shelter deduction. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Called the household ineligible on income and assets. Gross income is 112% of FPL and New Jersey's BBCE removes the asset test. The frozen $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." +us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Acknowledged that income is within limits but returned $0, citing incomplete shelter details. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Assumed no shelter costs because no rent or utilities were listed, whereas the frozen calculation uses the real estate taxes as shelter costs. Its $1,140 annual benefit assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." +us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Left the $12,000 of financial assistance out of countable income and applied no shelter deduction despite the property taxes. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Computed annual net income of $13,479 with no shelter deduction and returned $0 once 30% of net income exceeded the maximum allotment. It left out the $395.49/mo property-tax shelter deduction and did not apply New Jersey's $95/mo minimum benefit for eligible households." +us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Took a capped $712 shelter deduction from mortgage interest plus property taxes, which gave net income of $493 and $144/mo. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No SNAP value or explanation was returned, so nothing was available to score against the $1,140 reference." +us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Added $564/mo of mortgage interest to the property taxes ($1,560.17 shelter cost) and took the $744 capped deduction, which gave $158/mo. PolicyEngine's shelter cost is only the $995.75/mo of property taxes, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95/mo minimum." +us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed income exceeds SNAP limits for one person. Both income tests pass in the frozen derivation. Its $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." +us,scenario_056,snap,muse-wonder,prompt_ambiguity,categorical_eligibility,False,Applied only the earned-income and standard deductions and returned $0 once 30% of net income exceeded the maximum allotment. It left out the $395.49 property-tax shelter deduction and did not apply New Jersey's $95/mo minimum benefit for eligible households. +us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Applied the shelter cap using $564/mo of mortgage interest plus $996/mo of property taxes, which gave net income of ~$500 and $150/mo. The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted." +us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Claimed $17,839 exceeds the gross limit even though its own figure for the limit (~$19,000) is higher, then returned $0. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring." +us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Asserted that gross income exceeds 130% of FPL and net income exceeds the net limit. Gross income of $1,486.54/mo is 112% of FPL and net income in the frozen derivation is 61% of FPL. Its $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded." us,scenario_056,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_056,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model built the correct NJ gross income base of $5,839 but never applied New Jersey's $10,000 minimum filing threshold for single filers, which exempts this household's entire liability and drives the tax to $0.00. It compounded this by inventing a ""$2,000 standard deduction"" that New Jersey's gross income tax does not provide, then taxed the residual $2,839 at 1.4% to reach $39.75." us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model reproduced PolicyEngine's intermediate NJ main income tax exactly — $5,838.53 of gross income less the $1,000 regular exemption, taxed at the 1.4% first-bracket rate — and submitted that $67.75 as the final answer. It omitted the last step: New Jersey's minimum filing threshold exempts a single filer whose gross income is at or below $10,000, so the $67.74 preliminary liability is fully offset to $0.00." us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model swept the $12,000 of financial assistance into NJ gross income to reach $17,839, which New Jersey does not tax, and then claimed a $1,428 charitable deduction that the New Jersey gross income tax does not allow at all. Its residual $3,462 taxed at 1.4% yields $48.47, and it never reached the dispositive rule — the $10,000 single-filer minimum filing threshold that zeroes the liability on the true $5,838.53 gross income base." us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs. This is a contract failure rather than a substantive modeling error: the correct derivation is $5,838.53 of NJ gross income less the $1,000 regular exemption at 1.4% = $67.74, fully exempted by New Jersey's $10,000 minimum filing threshold to $0.00." us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $12,000 of financial assistance as New Jersey gross income, inflating the base to $13,772.82 after the $1,000 exemption, when NJ gross income is only the $1,498.53 alimony plus $4,340.00 self-employment income. It then abandoned its own $192.82 arithmetic and submitted $299.43, a figure no step in its reasoning produces, and it never applied New Jersey's $10,000 single-filer minimum filing threshold, which exempts the entire $67.74 preliminary liability to $0.00." -us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Applied the federal age 25 to 64 requirement for childless filers to the NJ EITC. It missed that NJ extends its refundable EITC to childless filers aged 18 and over using the federal formula. On $4,033.39 of net SE earnings, the reference awards $265.60." -us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"Correctly found the NJ childless EITC for ages 18 and up and the $4,033.39 earned income (net of half SE tax). It then multiplied the $308.55 phase-in amount by 40% to get $123.42. The reference's $265.60 is 40% of the $664 2026 maximum childless credit, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Said New Jersey has no state EITC or other refundable credit. In fact NJ has a refundable EITC that childless filers aged 18 and up can claim, so it never computed the $265.60 credit on $4,033.39 of net self-employment earnings." -us,scenario_056,state_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"Required NJ tax liability and qualifying children before any refundable credit could apply. The NJ EITC is refundable regardless of liability and is open to childless filers aged 18 and up, which yields $265.60 here." -us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used an under-21 cutoff, which was NJ's tax-year 2020 rule. Since 2021 the NJ childless EITC age floor has been 18, so the 20-year-old head qualifies on $4,033.39 of net SE earnings for the reference's $265.60." -us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Treated the head's unearned income and unmarried-partner status as disqualifying. Alimony and financial assistance do not count toward the investment-income limit. The $4,340 of SE income gives $4,033.39 of EITC earned income, which qualifies for the NJ childless EITC at age 18+ ($265.60)." -us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Required federal EITC eligibility and applied the federal under-25 exclusion. NJ separately extends its EITC to childless filers aged 18 and up, which produces the reference's $265.60." -us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"Correctly applied NJ's childless EITC for ages 18 and up with earned income of $4,033.39 and a $308.55 federal-formula phase-in amount, then took 40% to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Made NJ EITC depend on federal eligibility and applied the federal childless age 25 minimum. It missed NJ's extension to childless filers aged 18 and up, which gives the reference's $265.60." -us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Guessed that the head is someone's dependent and applied the federal age 25+ rule for childless filers. The head is the tax unit head, and NJ's EITC for childless filers aged 18+ applies, producing $265.60." -us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Counted the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65+, blind, or disabled. It also missed the refundable NJ EITC for childless filers aged 18+, which makes up the reference's $265.60." -us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Required federal EITC eligibility, which the 20-year-old fails because of the age 25 floor. It missed that NJ grants its EITC to childless filers aged 18 and up using the federal formula, giving $265.60." -us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"Computed the NJ EITC only as a share of the actual federal EITC ($0). It skipped NJ's separate pathway for childless filers aged 18 and up, which uses a federal-formula credit on $4,033.39 of earnings and yields $265.60." -us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Correctly used NJ's childless EITC for ages 18 and up, but phased it in on gross SE income of $4,340. EITC earned income is net SE earnings minus half of SE tax ($306.61), or $4,033.39. Its 40% × 7.65% shortcut also differs from the reference's $265.60, which is 40% of the $664 maximum and comes from capping the $308.55 phase-in amount." -us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Tied NJ EITC to a federal EITC of $0, which it got from the age-25 childless rule. It never applied NJ's age-18 floor for childless filers, under which the reference credit is $265.60." -us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Made the NJ EITC depend on federal EITC eligibility. NJ independently covers childless filers aged 18 and up using the federal formula, which gives the reference's $265.60." -us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Said no NJ refundable credit applies without checking the NJ EITC. It missed that the 20-year-old childless head, with $4,033.39 of net SE earnings, qualifies for NJ's childless EITC for ages 18 and up ($265.60)." -us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Correctly applied NJ's age-18 childless EITC on $4,033 of earnings and took 40% of the ~$309 federal-formula credit to get $123. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Correctly used NJ's EITC for childless filers aged 18 to 24 and took 40% of a ~$307 federal-formula phase-in amount to get $123. It used $4,008 instead of $4,033.39 of earned income. The reference's $265.60 is 40% of the $664 maximum, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Returned $0 without evaluating the NJ EITC. The childless 20-year-old head qualifies under NJ's age-18 floor on $4,033.39 of net SE earnings, which gives the reference's $265.60." -us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Concluded NJ credits are zero because the federal EITC is $0 and there are no children. NJ extends its EITC to childless filers aged 18 and up regardless of federal age eligibility, which yields $265.60." -us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Used an age-21 minimum, which was NJ's 2020 rule for childless filers. The floor has been 18 since tax year 2021, so the 20-year-old head qualifies for the reference's $265.60 NJ EITC." -us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Required federal EITC eligibility and mixed in the NJ Child Tax Credit's under-6 child rule. It missed that NJ's EITC covers childless filers aged 18 and up, producing $265.60." -us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"Applied the federal childless age range of 25 to 64 to the NJ EITC. NJ sets its own age floor of 18 for childless filers, so the head qualifies for the reference's $265.60." -us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Said the NJ childless EITC requires age 21+, which is the outdated 2020 rule. The floor has been 18 since tax year 2021, so the 20-year-old head qualifies for $265.60." -us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Found no NJ refundable credit and never evaluated NJ's EITC for childless filers aged 18 and up. On $4,033.39 of net SE earnings, that credit is $265.60 in the reference." -us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Required children or explicit low-income indicators. NJ's refundable EITC needs no children for filers aged 18+, and the head's $4,033.39 of net SE earnings yields $265.60." -us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Made refundable credits depend on positive NJ tax liability and dependents. The NJ EITC is refundable and open to childless filers aged 18 and up, giving the reference's $265.60." -us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Said no NJ EITC applies, missing NJ's eligibility rule for childless filers aged 18 and up. Under that rule, the 20-year-old head's $4,033.39 of net SE earnings yields $265.60." -us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Treated the $50 NJ property tax credit as refundable for a non-elderly, non-disabled homeowner. It is refundable only for filers who are 65+, blind, or disabled. It also missed the NJ childless EITC (ages 18 and up) that makes up the reference's $265.60." -us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Correctly used NJ's age-18 childless EITC, but computed a $332 federal-formula credit on gross SE income of $4,340. EITC earned income is net of the $306.61 half-SE-tax deduction, or $4,033.39. Taking 40% of the phase-in amount also differs from the reference's $265.60, which is 40% of the $664 maximum and comes from capping the phase-in amount." -us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Correctly applied NJ's age-18 childless EITC on earnings net of half SE tax, then took 40% of the $308.55 federal-formula credit to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Said the head does not qualify for the state EITC, missing NJ's extension to childless filers aged 18 and up. With $4,033.39 of net SE earnings, that credit is $265.60." -us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"Correctly found the 20-year-old childless head eligible for the NJ EITC and took 40% of the $308.55 federal-formula credit to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Correctly applied NJ's age-18 childless EITC to net earned income and took 40% of the $308.55 federal-formula credit to get $123.42. The reference's $265.60 is 40% of the $664 maximum childless credit, which the engine gets by capping the $308.55 phase-in amount at that scaled maximum." -us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,Said no state refundable credits apply without evaluating the NJ EITC. The childless 20-year-old head qualifies under NJ's age-18 floor for $265.60. -us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Derived the NJ EITC only from the actual federal EITC ($0). NJ computes a federal-formula credit for childless filers aged 18 to 24 who fail the federal age test, which yields $265.60." -us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,Assumed no federal EITC means no NJ EITC. NJ's separate pathway for childless filers aged 18 and up covers the 20-year-old head and produces $265.60. -us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"Correctly ruled out the $50 property tax credit as non-refundable for this head, but tied the NJ EITC to a federal EITC of $0. It missed NJ's eligibility for childless filers aged 18 and up, which gives the reference's $265.60." -us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Calculated the NJ EITC as 40% of an actual federal EITC of $0. It ignored NJ's age-18 floor for childless filers, under which the head gets the reference's $265.60." -us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"Treated federal EITC ineligibility as ruling out the NJ EITC. NJ extends the credit to childless filers aged 18 and up, which yields $265.60 on $4,033.39 of net SE earnings." -us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so there was no answer to score against the reference's $265.60 NJ EITC." -us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"Applied the federal age 25 to 64 range for childless filers and calculated the NJ EITC as 40% of $0. It missed NJ's extension to childless filers aged 18 and up, which produces $265.60." -us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Ruled out NJ refundable credits based on household composition alone. A childless single adult aged 18+ qualifies for the NJ EITC, giving $265.60 here." -us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Derived the NJ EITC from a federal EITC of $0 because the filer fails the age test. NJ separately covers childless filers aged 18 and up, which yields the reference's $265.60." -us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"Required qualifying children for NJ EITC eligibility. Filers aged 18+ with no children and earned income qualify, and the head's $4,033.39 of net SE earnings gives $265.60." -us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"Required qualifying children and positive NJ tax liability. The NJ EITC is refundable and covers childless filers aged 18 and up, producing the reference's $265.60." +us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Applied the federal age-25 minimum for childless workers and stopped there. It missed that NJ extends its 40% EITC to childless filers aged 18–24, which makes the 20-year-old eligible on $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"Applied the NJ age-18 extension and computed 40% × the 7.65% phase-in on $4,033.39 ($308.55) = $123.42. The engine instead credits 40% of the full $664 childless maximum ($265.60) after its reduction step on the −$5,882.05 AGI, so the model's phase-in-limited figure is $142.18 lower." +us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Asserted that New Jersey has no state EITC or refundable credits. NJ has a refundable EITC at 40% of the federal-formula credit, and it covers childless filers aged 18 and older, including this 20-year-old with SE earnings." +us,scenario_056,state_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"Tied refundable credits to having NJ tax liability and qualifying children. The NJ EITC is refundable regardless of liability and is available to childless filers aged 18 and older with earned income, as this 20-year-old has." +us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used an age-21 floor for the childless NJ EITC. NJ lowered the minimum age to 18, so the 20-year-old qualifies on $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Denied the credit because the head has no children, is an unmarried partner, and has unearned income. None of these bars the NJ childless EITC: the 20-year-old has $4,033.39 of net SE earnings, and alimony is not investment income." +us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,Required federal EITC eligibility and applied the under-25 exclusion. It missed NJ's extension of the childless EITC to filers aged 18–24. +us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"Applied the NJ age-18 childless rule and computed 40% × 7.65% × $4,033.39 = $123.42. The engine produces $265.60, which is 40% of the $664 childless maximum after its reduction step on the negative AGI, rather than 40% of the $308.55 phased-in amount." +us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Treated federal EITC eligibility (ages 25–64 for childless filers) as a precondition for the NJ EITC. NJ grants its own credit to childless filers aged 18–24, so the 20-year-old qualifies." +us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Assumed the head was a dependent or failed the childless age-25 test. NJ's EITC covers childless filers aged 18 and older, and nothing in the facts makes this head a dependent." +us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Credited the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65+, blind, or disabled. The model also missed the NJ childless EITC for filers aged 18 and older, which is the actual refundable credit here." +us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,Required federal EITC eligibility for the NJ credit and so excluded the 20-year-old. It missed NJ's statutory extension of the childless EITC to ages 18–24. +us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,Computed the NJ EITC as 40% of a $0 federal EITC. It missed that NJ computes a federal-formula credit for childless filers aged 18–24 who fail only the federal age test. +us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Applied the NJ age-18 rule but used gross SE income of $4,340 as earned income instead of $4,033.39 (after the $306.61 deductible half of SE tax). It got $132.80, while the engine credits 40% of the $664 childless maximum ($265.60)." +us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,Tied the NJ EITC to a federal EITC that is $0 because of the age-25 floor. It missed NJ's separate eligibility for childless filers aged 18–24. +us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Conditioned NJ EITC eligibility on federal EITC eligibility. It ignored that NJ extends the childless credit to filers aged 18–24, which covers this 20-year-old with SE earnings." +us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Returned $0 without identifying the NJ EITC. It missed that NJ's childless EITC at 40% of the federal formula applies to a 20-year-old with $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Applied NJ's age-18 rule and took 40% of the ~$309 phased-in credit on $4,033, giving $123. The engine's $265.60 is 40% of the full $664 childless maximum, produced by its reduction step on −$5,882.05 AGI." +us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Applied the NJ 18–24 look-alike EITC at 40% to a phased-in credit (using $4,008 of earnings rather than $4,033.39), giving $123. The engine's $265.60 is 40% of the $664 childless maximum, not of the phased-in amount." +us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Gave $0 with no supporting rule. It missed the NJ childless EITC that applies to filers aged 18 and older with earned income, which this 20-year-old self-employed head meets." +us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Derived the NJ EITC from a $0 federal EITC. It missed NJ's extension of childless eligibility to ages 18–24, which covers this 20-year-old." +us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Applied an age-21 minimum to NJ refundable credits. NJ's childless EITC minimum age is 18, so the 20-year-old qualifies." +us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Required federal EITC eligibility or a child under 6, mixing up the NJ EITC with the NJ Child Tax Credit. It missed that the NJ EITC covers childless filers aged 18–24." +us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,Applied the federal 25–64 childless age range to the NJ EITC. NJ independently extends the credit to childless filers aged 18–24. +us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Stated that NJ's childless EITC requires age 21 or older. NJ lowered the threshold to 18, so the 20-year-old head qualifies." +us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Found no refundable NJ credit. It overlooked the NJ EITC for childless filers aged 18 and older, which applies to the head's $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,Treated the absence of children as ruling out NJ refundable credits. The NJ EITC is available to childless filers aged 18 and older with earned income. +us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,Required dependents and positive NJ tax liability for a refundable credit. The NJ EITC is refundable without liability and covers childless filers aged 18 and older. +us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Declared that no NJ EITC applies. It missed NJ's childless EITC eligibility for filers aged 18–24, which this 20-year-old with SE earnings meets." +us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Treated the $50 NJ property tax credit as refundable for a non-elderly, non-disabled homeowner, when it is refundable only for filers who are 65+, blind, or disabled. The model also missed the NJ childless EITC for filers aged 18 and older." +us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Applied the 7.65% phase-in to gross SE income of $4,340 ($332) instead of net earned income of $4,033.39 (after the $306.61 half of SE tax). It got $132.80, while the engine credits 40% of the $664 childless maximum ($265.60)." +us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Applied the NJ age-18 extension and took 40% of the 7.65% phase-in on $4,033.39, giving $123.42. The engine credits 40% of the $664 childless maximum ($265.60) after its reduction step on the −$5,882.05 AGI." +us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Concluded that the head does not qualify for the NJ EITC. NJ extends the childless credit to filers aged 18 and older, and the head has $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"Applied the NJ childless extension and took 40% of the $308.55 phased-in federal-formula credit, giving $123.42. The engine's $265.60 is 40% of the full $664 childless maximum." +us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Applied the NJ age-18 rule and 40% match to the phased-in credit on net earnings, giving $123.42. The engine instead credits 40% of the $664 childless maximum ($265.60) after its reduction step on negative AGI." +us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"Asserted that no state credit qualifies. It missed the NJ childless EITC for filers aged 18 and older, which applies to this 20-year-old with SE earnings." +us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,Derived the NJ EITC as a share of a $0 federal EITC. It missed NJ's separate eligibility for childless filers aged 18–24. +us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,Treated the lack of a federal EITC as eliminating the NJ EITC. NJ computes the credit for childless filers aged 18–24 who are excluded federally only by age. +us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,Correctly ruled out the $50 property tax credit but computed the NJ EITC from a $0 federal EITC. It missed that NJ extends the childless EITC to filers aged 18–24. +us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,Computed the NJ EITC as 40% of a $0 federal EITC. It missed that NJ grants the childless credit to filers aged 18–24 who fail only the federal age test. +us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"Treated federal EITC ineligibility as disqualifying for NJ. NJ's EITC independently covers childless filers aged 18 and older, including this 20-year-old." +us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so no answer could be scored." +us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"Applied the federal 25–64 childless age range to the NJ EITC. NJ extends the credit to childless filers aged 18–24, which makes the 20-year-old eligible." +us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Ruled out NJ credits because of household composition. The NJ EITC is available to a single childless filer aged 18 or older with earned income, as this head is." +us,scenario_056,state_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"Found no eligibility basis for an NJ refundable credit. It missed NJ's childless EITC for filers aged 18 and older, which applies to $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,Computed the NJ EITC from a federal EITC that is $0 because of the age-25 floor. It missed NJ's extension of the childless credit to ages 18–24. +us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"Required qualifying children for the NJ EITC. NJ grants the EITC to childless filers aged 18 and older with earned income, and the head has $4,033.39 of net SE earnings." +us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,Required children and NJ tax liability for refundable credits. The NJ EITC is refundable without liability and covers childless filers aged 18 and older with earned income. us,scenario_056,tanf,glm-5.3,llm_error,categorical_eligibility,False,"The model read the $12,000 `financial_assistance` input as if it were reported TANF receipt and echoed it back verbatim as the computed benefit, when the question asks what PolicyEngine computes for `tanf` from the household facts. It never applied TANF's categorical requirement that the assistance unit contain a dependent child — New Jersey's WorkFirst NJ TANF covers only units with a child under 18 (or 18 and in school full time), while childless adults fall to WFNJ General Assistance, which is not TANF — so this childless 20-year-old head yields $0. Its answer of exactly 12000 is arithmetically identical to the input line it copied, confirming no eligibility test or benefit computation was performed at all." us,scenario_056,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no `tanf` entry in its `outputs` object and no explanation, so the required key was absent rather than substantively wrong. The correct derivation is that WorkFirst NJ TANF requires an assistance unit containing a dependent child, and this single 20-year-old head with zero children fails that categorical test, producing $0." us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly placed earned income of about $9,604 on the plateau, below the phase-out. It then cut its own ~$659 maximum to $636 with an invented 'lookup-table rounding' adjustment. On the plateau the credit equals the 2026 statutory maximum of $664, with no table reduction." @@ -5004,53 +5058,54 @@ us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model computed the correct $1,469.46 from $9,604.40 × 15.3%, then added roughly $82.60 of 0.9% Additional Medicare Tax by treating $400 as the AMT threshold; the AMT threshold is $250,000 of combined earnings for joint filers, and the prompt explicitly excludes Additional Medicare Tax from this output. Its arithmetic is also self-contradictory — the stated components sum to $1,552.06, not the $1,531.79 submitted — where the correct answer is the unadjusted $1,469.47." us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model fabricated a ""$12,000 threshold for self-employment tax liability"" and zeroed the tax; no such filing floor exists, and the only threshold in §1402(b) is $400 of net earnings, which $9,604.40 clears. Applying the 15.3% rate to that base gives $1,469.47." us,scenario_057,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model derived the correct liability — 0.153 × $9,604.40 = $1,469.47 — and then reported half of it, confusing the §164(f) above-the-line deduction for one-half of SE tax with the tax owed; that deduction reduces AGI and never reduces the §1401 liability itself. Its submitted $734.76 is not even the correct half ($734.74), and the requested output is the full $1,469.47." -us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Never computed or counted the head's $602.58/month SSI as SNAP unearned income, so it used about $870/month of gross income instead of $1,472.42. It also guessed a $540 maximum allotment instead of $546, landing at $380/month instead of $219." -us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted SSI at the couple rate of about $1,099/month as if both spouses were SSI recipients. The engine's SSI is $602.58/month (the $994 individual rate for the head, minus $391.42 of countable earnings). That overstated net income at $1,586 instead of $1,090.08 and cut the benefit to $70/month instead of $219." -us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Wrongly cut self-employment income to about $809 using a self-employment-tax factor, then called the household ineligible without any calculation. With $1,472.42/month of gross income (including $602.58 of SSI), net income is $1,090.08, below the $1,762.50 net-income limit, so the household is eligible for $219/month." -us,scenario_057,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Asserted a zero benefit without computing net income. The $912 in assets passes the asset test, and net income of $1,090.08 is 62% of the poverty guideline, so the household gets $546 − $327 = $219/month." -us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month SSI out of SNAP income. It also subtracted a monthly $219 standard deduction from annual income. Its own arithmetic produced $3,990, but it then submitted $6,240 with no justification." -us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Left SSI out of SNAP income and treated the $38/year of disability benefits as $456/year of unearned income. It then raised its own $4,536 estimate to $5,736 for medical deductions, which don't apply because the $40/year of OTC expenses is far below the $35/month threshold." -us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Claimed net income was near zero, ignoring the $866.67/month of self-employment income and the $602.58/month of SSI. It also used an invented $304/month maximum allotment instead of the $546 FY2026 two-person maximum." -us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Counted SSI of $1,098.58/month at the couple rate. The engine pays only the head $602.58/month at the $994 individual rate, so net income is $1,090.08, not $1,586, and the benefit is $219/month, not $70." -us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Left the $602.58/month of SSI out of SNAP income. It also took an unlisted 40% business-expense deduction off the $10,400 of self-employment income, which SNAP counts in full here, and used a $535 maximum allotment instead of $546." -us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Reduced self-employment income with a self-employment-tax deduction that SNAP doesn't use, left out the $602.58/month of SSI, and treated the $38/year of disability benefits as $38/month. Its stated net income of about $2,000/month doesn't match its $297/month benefit." -us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Added SSI of about $1,099/month at the couple rate. The engine pays only the head $602.58/month at the individual rate, so net income is $1,090.08, not $1,586, and the benefit is $219/month, not $70." -us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Used only earnings and disability benefits (about $870/month) as gross income and never added the head's $602.58/month of SSI. That understated net income at $488.51 instead of $1,090.08. It also used $536 and $208 in place of the $546 maximum and the $209 standard deduction." -us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Left the $602.58/month of SSI out of SNAP gross income, so its net income was $489.34 instead of $1,090.08. It also used a $204 standard deduction and a $536 maximum instead of $209 and $546." -us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted SSI of $1,067.17/month at the couple rate instead of the head's $602.58 individual-rate SSI, and treated the $38/year of disability benefits as $38/month. That pushed net income to $1,594.50 and the benefit down to $57.65/month." -us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Never added the head's $602.58/month of SSI to SNAP income, so net income came out at $492.50 instead of $1,090.08 and the benefit at $398/month instead of $219." -us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Counted $13,122.96/year of SSI at the couple rate instead of the head's $602.58/month individual-rate SSI. It also took a $60 excess medical deduction that doesn't exist (OTC expenses of $40/year are below the $35/month threshold) and used a $6,600 annual maximum instead of $546/month." -us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Declared the household ineligible without any computation. Net income of $1,090.08 is well under the $1,762.50 net-income limit, which yields $219/month for most of the year." -us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Included SSI at the roughly $1,099/month couple rate instead of the head's $602.58/month individual-rate SSI. That overstated net income and gave $76/month instead of $219." -us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted $12,729/year of unearned income, which is couple-rate SSI for both spouses. The engine's SSI is only the head's $602.58/month ($7,230.96/year), so net income is $1,090.08/month and the benefit is $219/month." -us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted that income and assets block SNAP with no calculation. The $912 in assets passes the asset test, and net income of $1,090.08 is 62% of the poverty guideline, which yields $219/month." -us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Gave $3,216 ($268/month) with no derivation. The correct calculation counts the $602.58/month of SSI alongside $866.67 of earnings, giving net income of $1,090.08 and $219/month (rising to $232.74 for October–December), for $2,669.22." -us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave $3,384 ($282/month) with no derivation. This understates countable income: including the $602.58/month of SSI gives net income of $1,090.08 and a $219/month allotment for most of the year, totaling $2,669.22." -us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Built gross income from only the $10,400 of self-employment income and $38 of disability benefits and never counted the head's $602.58/month of SSI. It also used a $200 standard deduction instead of $209, giving net income of $496.50 instead of $1,090.08." -us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted benefit-receipt input set to 0 instead of a benefit to compute. With take-up assumed, the household's net income of $1,090.08 yields $219/month." -us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,100/month at the couple rate instead of the head's $602.58/month individual-rate SSI, and used a $230 standard deduction instead of $209. That inflated net income to $1,566 and cut the benefit to $66/month." -us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed counted income was too high without computing it. Gross income of $1,472.42 is 84% of the poverty guideline and net income of $1,090.08 passes the 100% net-income test, which yields $219/month." -us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SNAP as a receipt amount that had to be given in the facts instead of computing it under the take-up assumption. The household's net income of $1,090.08 produces a $219/month allotment." -us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month of SSI out of SNAP income, using $10,438/year of gross income instead of $17,669. That understated net income and produced about $400/month instead of $219." -us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,Applied a medical deduction that doesn't exist ($40/year of OTC expenses is below the $35/month threshold) and left the $602.58/month of SSI out of countable income. The result was $340/month instead of $219. -us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"A $70/month benefit implies net income of about $1,586, which counts couple-rate SSI of about $1,099/month. The engine pays only the head $602.58/month at the individual rate, giving net income of $1,090.08 and $219/month." -us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI at the couple rate (about $1,099/month), giving a $70/month allotment. The head's individual-rate SSI of $602.58 gives net income of $1,090.08 and $219/month." -us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Its $23,621 of SNAP income includes about $13,183/year of couple-rate SSI. The engine's SSI is the head's $602.58/month ($7,230.96/year), so annual net income is about $13,081, not $19,033." -us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Computed net income of $487.67/month from earnings and disability benefits alone and left out the head's $602.58/month of SSI, which brings net income to $1,090.08 and the benefit down to $219/month." -us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Its $842.10 answer reflects couple-rate SSI of about $1,099/month in net income ($1,586). The head's individual-rate SSI is $602.58/month, so net income is $1,090.08 and the benefit is $219/month." -us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Its net income of $1,586.08 includes couple-rate SSI of about $1,099/month instead of the head's $602.58/month individual-rate SSI. That overstated net income by about $496 and cut the allotment to $70/month." -us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that income and assets exceed SNAP limits without computing them. Assets of $912 and net income of $1,090.08 both pass, giving $219/month." -us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Its monthly gross of about $1,928 includes roughly $1,058 of couple-rate SSI instead of the head's $602.58 individual-rate SSI, so net income came out at $1,551 instead of $1,090.08. It also used a $204 standard deduction instead of $209." -us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Its monthly gross of about $1,955 includes about $1,085 of couple-rate SSI. The head's individual SSI is $602.58, so gross income is $1,472.42 and net income is $1,090.08. It also used a $536 maximum and a $204 standard deduction instead of $546 and $209." -us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Counted $13,123/year of couple-rate SSI instead of the head's $7,230.96/year ($602.58/month) of individual SSI, which inflated net income to $18,973/year and shrank the benefit to $860." -us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Subtracted the $209 monthly standard deduction from annual income in the same calculation as an annual $6,600 maximum, and left out the $602.58/month of SSI. The correct monthly calculation gives net income of $1,090.08 and $219/month." -us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"Used about $870/month of gross income and never added the head's $602.58/month of SSI. That gave net income of $488 instead of $1,090.08 and a $399/month benefit instead of $219." -us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score." -us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Counted $13,183/year of couple-rate SSI. The engine's SSI is the head's $602.58/month at the $994 individual rate, so net income is $1,090.08, not $1,586.08, and the benefit is $219/month, not about $70." -us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Claimed $912 of bank assets exceeds a $4,500 limit, which is backwards: $912 is below the limit, and the household also has categorical eligibility through SSI. With the asset test passed, net income of $1,090.08 yields $219/month." -us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Counted only $867 of earnings and $3 of disability benefits as income and omitted the head's $602.58/month of SSI. That gave net income of $488 instead of $1,090.08 and about $400/month instead of $219." -us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,Wrongly excluded all income of SSI-eligible members from SNAP. SNAP counts their self-employment income and the $602.58/month of SSI. It also used a $635 maximum allotment instead of $546. -us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Concluded the household gets no SNAP without any income test. Gross income of $1,472.42 (84% of the poverty guideline) and net income of $1,090.08 (62%) both pass, giving $219/month." +us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It counted only the $10,400 of self-employment income and $38 of disability benefits (about $870/month). It left out the $602.58/month SSI that PolicyEngine computes for the blind, disabled head, which is counted unearned income for SNAP. That put its net income near $485 instead of $1,090.08, so it overstated the monthly benefit by about $160." +us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It counted SSI at about $1,099/month (the $1,491 couple FBR minus combined countable earnings), while PolicyEngine's SSI is $602.58/month (the $994 individual FBR minus the head's $391.42 countable income). This raised net income to $1,586 instead of $1,090.08 and the contribution to about $476 instead of $327. It also used the $546 maximum for all 12 months and missed the October increase to $558.24." +us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"It recognized that countable income was very low but still returned $0, citing net income 'slightly exceeding' the limit. In fact, net income of $1,090.08 is 62% of the $1,762.50 poverty line, and SSI receipt makes the household categorically eligible. It also wrongly cut self-employment income with a 92.35% SE-tax factor, which SNAP does not apply." +us,scenario_057,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"It denied SNAP even though the $912 in bank assets is far below any limit, SSI receipt confers categorical eligibility, and net income of $1,090.08 passes the net income test. The correct benefit is $219/month for January–September and $232.74/month for October–December." +us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It left out the head's $602.58/month SSI and subtracted a monthly standard deduction from annual income. It then dropped its own computed $3,990 and submitted $6,240, close to the full maximum allotment, with no supporting arithmetic. The correct contribution is $327/month on net income of $1,090.08." +us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It left out the $602.58/month SSI and inconsistently treated the $38 annual disability benefits as $456. It then raised its own $4,536 estimate to $5,736 for medical deductions that do not exist, since $40/year of over-the-counter costs is below the $35/month threshold." +us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It assumed net income was near zero and used a $304 two-person maximum allotment. The FY2026 two-person maximum is $546, and net income is $1,090.08 once the $602.58 SSI and $866.67 of self-employment income are counted, which gives a $327 contribution." +us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"It counted SSI of $1,098.58/month at the couple rate, while PolicyEngine's SSI is $602.58 ($994 individual FBR minus $391.42). That inflated net income to $1,586 and cut the benefit to $70/month. It also kept the $546 maximum through December and missed the FY2027 $558.24 maximum for October–December." +us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"It left out the head's $602.58/month SSI. It also applied a 40% cost-of-production deduction to self-employment income, while PolicyEngine counts the full $866.67/month as earned income. It used a $535 maximum instead of $546, which drove net income down to $205 instead of $1,090.08." +us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It cut self-employment income by a self-employment-tax deduction that SNAP does not use and left out the $602.58/month SSI. It then asserted net income of about $2,000/month and a $297 benefit with no consistent arithmetic. The correct net income is $1,090.08, which yields $219/month." +us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"It counted SSI of about $1,099/month at the couple rate instead of PolicyEngine's $602.58/month at the individual rate. That overstated net income by about $496 (giving $1,586 vs $1,090.08) and gave a $70 monthly benefit. It also ignored the higher October–December FY2027 maximum." +us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"It counted only self-employment and disability income (about $870/month) and left out the $602.58/month SSI that is unearned SNAP income. This gave net income of $488.51 instead of $1,090.08. It also used a $536 maximum and a $208 standard deduction instead of $546 and $209." +us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It left out the head's $602.58/month SSI from gross income, so net income came out at $489.34 instead of $1,090.08 and the benefit at about $389/month instead of $219. It also used outdated $204 standard-deduction and $536 maximum values." +us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It counted SSI of $1,067.17/month at about the couple rate instead of PolicyEngine's $602.58, and treated the $38 annual disability benefits as monthly income. That pushed net income to $1,594.50, and it applied an outdated $536 maximum for the whole year." +us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"It left out the $602.58/month SSI the head receives, so it computed net income of $492.50 instead of $1,090.08. As a result, its contribution was $147.75 instead of $327." +us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It counted SSI of $13,122.96/year at the couple rate instead of PolicyEngine's $7,230.96 ($602.58/month). It also added a $60 medical deduction even though $40/year of over-the-counter costs never exceeds the $35/month threshold, and it used a made-up $6,600 annual maximum." +us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"It declared the household ineligible, but the household is categorically eligible through SSI receipt. Net income of $1,090.08 is only 62% of the poverty line, which produces $219/month for January–September and $232.74/month for October–December." +us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Its $76/month benefit implies about $1,567 of net income, which reflects SSI at the roughly $1,099 couple rate. PolicyEngine counts $602.58/month of SSI, which gives net income of $1,090.08 and a $219 benefit." +us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It counted $12,729/year of SSI plus disability benefits (about $1,061/month), while PolicyEngine's SSI is $602.58/month, about $7,231/year. That overstated unearned income by about $455/month and cut the benefit to about $65/month." +us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"It concluded the benefit is zero 'under Louisiana rules', but the household passes every test: assets of $912, categorical eligibility through SSI, and net income of $1,090.08 under the $1,762.50 limit. The $546 maximum minus the $327 contribution leaves $219/month." +us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"It submitted $268/month with no income or deduction computation. Correctly counting $1,472.42 of gross income (including $602.58 of SSI), minus $173.33 earned-income and $209 standard deductions, gives a $219 benefit for January–September and $232.74 for October–December." +us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It gave $282/month with no derivation. The correct calculation counts the head's $602.58 SSI with self-employment and disability income to reach net income of $1,090.08 and a $327 contribution against the $546 maximum." +us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"It built gross income from only the $10,400 of self-employment income and $38 of disability benefits, leaving out the $602.58/month SSI. It also used a $200 standard deduction instead of $209, so net income came out at $496.50 instead of $1,090.08." +us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It treated SNAP as an unlisted benefit receipt and set it to zero, ignoring the instruction to assume take-up and compute eligibility. The household is eligible for $219/month for January–September and $232.74/month for October–December." +us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It counted about $1,100/month of SSI at the couple rate instead of PolicyEngine's $602.58, and used a $230 standard deduction instead of $209. That produced net income of $1,566 instead of $1,090.08 and a $66 benefit." +us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"It claimed counted income was too high. Gross income of $1,472.42 is 84% and net income of $1,090.08 is 62% of the poverty line, and SSI receipt confers categorical eligibility, so the household receives $219–$232.74 per month." +us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It set SNAP to zero because no benefit amount was given, instead of computing the benefit under assumed take-up. With $1,090.08 of net income, the $546 maximum minus the $327 contribution gives $219/month." +us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"It computed annual gross income as just $10,438 and left out the head's $7,230.96 of annual SSI, which is counted unearned income for SNAP. Net income was therefore understated by about $7,231/year, and the benefit was overstated by about $2,100." +us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It claimed medical deductions that do not apply, since $40/year of over-the-counter costs is below the $35/month threshold, and it left out the $602.58/month SSI from income. Its $340/month figure does not match the correct net income of $1,090.08 and the $219 benefit." +us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Its $70/month benefit implies net income of about $1,586, which reflects SSI at the roughly $1,099 couple rate instead of PolicyEngine's $602.58/month. It also applied $546 for all 12 months instead of $558.24 for October–December." +us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It counted SSI at about the $1,099/month couple rate, which gave a $70 allotment. PolicyEngine counts $602.58/month of SSI, which gives net income of $1,090.08 and a $219 allotment, rising to $232.74 for October–December." +us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It estimated SSI of about $13,183/year at the couple rate (giving $23,621 of income), while PolicyEngine's SSI is $7,230.96/year at the individual rate. That overstated net income by about $5,950/year, and it also used a flat annual maximum that misses the FY2027 increase in October–December." +us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"It computed net income of $487.67 from self-employment and disability income alone and left out the head's $602.58/month SSI. The correct net income is $1,090.08, with a $327 contribution." +us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Its $842.10 result implies about $1,586/month of net income, which comes from counting SSI at the roughly $1,099 couple rate instead of PolicyEngine's $602.58/month. It also ignored the October–December FY2027 maximum of $558.24." +us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"It reached net income of $1,586.08 by counting SSI of about $1,098.58/month at the couple rate. PolicyEngine's SSI is $602.58/month ($994 individual FBR minus $391.42), which gives net income of $1,090.08 and a $219 benefit." +us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"It asserted that income and assets exceed the limits, but the $912 in assets is far below any limit, SSI receipt confers categorical eligibility, and net income of $1,090.08 yields a positive $219/month benefit." +us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It counted about $1,058/month of SSI (monthly gross of $1,928) instead of PolicyEngine's $602.58, and used an outdated $204 standard deduction. That raised net income to about $1,551 and cut the benefit to $71/month." +us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It counted about $1,085/month of SSI at roughly the couple rate instead of PolicyEngine's $602.58, and used a $536 maximum and a $204 standard deduction. That gave net income of $1,578 and a $63 benefit." +us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"It counted $13,123/year of SSI at the couple rate instead of PolicyEngine's $7,230.96/year. That overstated annual net income by about $5,900, and it used a flat $6,552 annual maximum that misses the October–December FY2027 increase." +us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"It subtracted a $209 monthly standard deduction from annual income, mixing periods, and left out the head's $602.58/month SSI. Its $4,155 result reflects neither the monthly net income of $1,090.08 nor the $219/month benefit." +us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"It counted gross income of only about $870/month and left out the $602.58/month SSI that PolicyEngine computes for the blind, disabled head. That understated net income at about $488 instead of $1,090.08." +us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value or explanation, so there was nothing to score against the $2,669.22 annual benefit." +us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"It estimated SSI of $13,183/year from the $1,491 couple FBR, while PolicyEngine's SSI is $602.58/month ($994 individual FBR minus $391.42). That gave net income of $1,586.08 instead of $1,090.08, and it applied the $546 maximum to all 12 months." +us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It claimed the $912 in bank assets exceeds an asset limit that it itself put at about $4,500, which is self-contradictory. The household passes the asset test, is categorically eligible through SSI, and receives $219–$232.74 per month." +us,scenario_057,snap,muse-wonder,prompt_ambiguity,age_disability,False,"It counted about $1,102/month of SSI at the couple rate instead of PolicyEngine's $602.58/month. That inflated net income to about $1,590 instead of $1,090.08 and cut the benefit to $69/month." +us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"It counted only $867 of earnings and $3 of disability benefits and left out the $602.58/month SSI. That gave net income of about $488 instead of $1,090.08 and a benefit of about $400 instead of $219." +us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It wrongly excluded all income of SSI-eligible members from SNAP. SNAP counts the $866.67 of self-employment income, $3.17 of disability benefits, and $602.58 of SSI. It also used a made-up $635 maximum allotment instead of $546." +us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"It found no positive benefit, but the household is categorically eligible through SSI and passes both income tests. Net income of $1,090.08 gives a $219/month benefit for January–September and $232.74 for October–December." us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as covering low-income adults through a standalone income test. It failed to apply the controlling Medicaid exclusion: the 20-year-old spouse is Medicaid-eligible in the adult category and therefore is not eligible for CHIP, regardless of being below an asserted CHIP income threshold." us,scenario_057,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted that income exceeds Louisiana's Medicaid limits, but household MAGI is 0.45 x FPL — roughly $10,438 against a 138% FPL expansion cutoff near $29,200 for a household of two in 2026 — so it compared the $10,400 self-employment figure against a threshold far lower than the one the ACA adult expansion actually applies. It also gave the spouse's disability status weight while never running the only test that governs a 20-year-old in an expansion state: MAGI at or below 138% FPL under the ADULT category." us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid as requiring an affirmative qualifying attribute in the fact pattern and defaulted to 0 in its absence, missing that Louisiana's ACA adult expansion makes age 19–64 plus MAGI at or below 138% FPL sufficient on its own. The listed facts fully determine eligibility — spouse age 20, non-dependent, household MAGI at 0.45 x FPL — so the ADULT expansion category resolves to eligible without any additional detail." @@ -5065,53 +5120,54 @@ us,scenario_057,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,Fa us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model said being disabled satisfies the under-65 Medicare pathway. That pathway requires 24 months of SSDI entitlement (or ESRD/ALS), not disabled status alone. The spouse has no SSDI months, so under-65 eligibility fails." us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented WIC coverage for women up to age 60 and treated disability as a route to postpartum or other categorical eligibility. The spouse is not pregnant, breastfeeding, or postpartum, so satisfying an income limit cannot establish WIC eligibility." us,scenario_057,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated being a 20-year-old disabled woman as a qualifying WIC category. Age, sex, and disability do not replace the required pregnancy, breastfeeding, or postpartum status, none of which is present." -us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"The model submitted a placeholder 0 and never computed SSI. The head alone qualifies at the $994/month individual rate. After the $20 general exclusion, the $65 earned-income exclusion and the one-half earnings exclusion, the head has $391.42/month of countable income, leaving a $602.58/month ($7,231/year) payment." -us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated both spouses as an SSI-eligible couple and used the $1,491/month ($17,892/year) couple rate. Under PolicyEngine's rules the spouse is not SSI aged/blind/disabled, so the head gets the $994/month individual rate. The spouse's $2/month of income falls below the deeming threshold. The head's countable income is $391.42/month, giving $7,231/year." -us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model left out the $20 general exclusion and the one-half exclusion of remaining earnings, counting $9,620 of self-employment income in full. It also claimed that $10,076 of countable income exceeds the $16,980 couple rate, which its own numbers contradict. The correct countable income is $391.42/month against the head's $994 individual rate, for $7,231/year." -us,scenario_057,ssi,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model declared the household SSI-ineligible without any calculation. The head is blind and disabled with $912 in resources, under the $2,000 individual limit. After exclusions, the head's $866.67/month of earnings leaves only $391.42 countable against the $994 individual rate, so the head receives $7,231/year." -us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model treated the spouse as SSI-eligible and used a couple rate of about $17,604/year. It also applied the $20 general exclusion twice, once against earnings and again against unearned income. The head is the only eligible individual and is paid $994/month minus $391.42 countable income, or $7,231/year." -us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model used an inflated couple rate of $1,538/month for both spouses, and its final figure does not follow from its own subtraction. The spouse is not SSI aged/blind/disabled, so the head gets the $994/month individual rate minus $391.42 of countable income, or $7,231/year." -us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"The model counted about $9,600 of self-employment income without the one-half exclusion of earnings left after the $20 and $65 exclusions. Correctly excluded, countable earnings are $391.42/month, well under the head's $994 individual rate, so the head receives $7,231/year rather than $0." -us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model's exclusion arithmetic was nearly correct, but it subtracted countable income from the $1,491 couple rate because it treated the spouse as eligible. The spouse fails the SSI aged/blind/disabled test, so the head's benefit is $994 − $391.42 = $602.58/month, or $7,231/year." -us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"The model used a $1,486/month couple rate for both spouses. Its final pass also applied the $20 general exclusion on an annual basis to unearned income and dropped the unused portion that should carry over to earnings. The head alone is eligible at $994/month with $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model's own calculation produced a positive $12,590 couple benefit, yet it submitted 0. The calculation itself was also wrong: it used the $1,450 couple rate and did not carry the unused general exclusion over to earnings. The head alone qualifies at $994/month minus $391.42 countable income, or $7,231/year." -us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model applied the $1,491/month couple rate on the premise that both spouses are SSI-eligible. Under PolicyEngine's rules the spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse. The benefit is $994 − $391.42 = $602.58/month, or $7,231/year." -us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model paid the spouse a nearly full $991/month individual benefit. The spouse fails the SSI aged/blind/disabled test and gets $0. The model also overstated the head's countable income at about $430 when the correct figure is $391.42. The head alone receives $602.58/month, or $7,231/year." -us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model subtracted $392.42 of countable income from an estimated $1,483 couple rate, treating the spouse as eligible. Only the head qualifies, so the individual rate of $994/month applies, less $391.42 of the head's own countable income, for $7,231/year." -us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model used the $1,486 couple rate for both spouses. It also counted $18 of unearned income instead of letting the $20 general exclusion absorb the $3.17 and carry the rest over to earnings. The head alone is eligible at $994/month with $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model applied the $1,491/month couple rate as though both spouses were SSI-eligible. The spouse is not SSI aged/blind/disabled, so the head receives the $994 individual rate minus $391.42 of countable income, or $7,231/year." -us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated both spouses as eligible and used a $1,486/month couple rate ($17,832/year). Only the head passes the SSI aged/blind/disabled test, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"The model claimed the head's earnings rule out SSI. Applying the $20 and $65 exclusions and then excluding half the remainder leaves only $391.42/month countable against the $994 individual rate, so the head receives $7,231/year." -us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model applied the couple maximum benefit on the premise that both spouses qualify. The spouse is not SSI aged/blind/disabled, so the head receives the $994/month individual rate minus $391.42 of countable income, or $7,231/year." -us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from a $17,400 couple rate, treating both spouses as eligible. Only the head qualifies, so the individual rate of $11,928/year ($994/month) applies, less $4,697 of countable income, giving $7,231." -us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"The model claimed income and resources exceed the SSI limits. The $912 in the bank is under the $2,000 individual limit, and countable income after exclusions is $391.42/month, below the $994 individual rate, so the head receives $7,231/year." -us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model computed an eligible-couple benefit because it treated the spouse as SSI-disabled. Only the head passes the SSI aged/blind/disabled test, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated both adults as an eligible couple and paid a couple-rate benefit. The spouse is not SSI aged/blind/disabled, so the head alone receives $994 − $391.42 = $602.58/month, or $7,231/year." -us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"The model claimed SSI is not simulated for this household. The blind, disabled head meets the resource test and has $391.42/month of countable income against the $994 individual rate, for $7,231/year." -us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model assumed the head's $10,400 of earnings exceeds the SSI limits and treated the absence of listed SSI receipt as zero benefit, although the prompt says to assume take-up. After exclusions, countable income is $391.42/month against the $994 individual rate, for $7,231/year." -us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"The model used a couple rate of about $1,490.56 because it treated the spouse as SSI-eligible. Only the head qualifies, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model said the household's assets exceed the SSI limit, but $912 is under the $2,000 individual limit. It also treated $10,400 of earnings as disqualifying even though only $391.42/month is countable after exclusions, against the $994 individual rate. The head receives $7,231/year." -us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model set SSI to 0 because no SSI receipt was listed, ignoring the instruction to assume take-up. The blind, disabled head qualifies at $994/month minus $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple rate, treating both spouses as eligible. The spouse fails the SSI aged/blind/disabled test, so the head is paid the $11,928/year individual rate less $4,697 of the head's own countable income, giving $7,231." -us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model applied a couple rate (the 2025 level of about $1,450) for both spouses. Only the head is SSI-eligible, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model reduced the 2026 couple maximum on the premise that both spouses are SSI-disabled. The spouse is not SSI aged/blind/disabled, so the head alone receives $994 − $391.42 = $602.58/month, or $7,231/year." -us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model computed an eligible-couple benefit of $13,183. Only the head passes the SSI aged/blind/disabled test, so the individual rate of $994/month applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple maximum, treating the spouse as eligible. The head is an individual with an ineligible spouse, so the benefit is the $11,928 individual rate minus $4,697, or $7,231." -us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model used the $1,491 couple rate against $392.42 of countable income. The spouse is not SSI aged/blind/disabled, so the head receives $994 − $391.42 = $602.58/month, or $7,231/year." -us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model reduced the $1,491/month couple rate, treating both spouses as eligible. Only the head qualifies, so the $994 individual rate minus $391.42 of countable income gives $7,231/year." -us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple maximum. The spouse fails the SSI aged/blind/disabled test, so the head's benefit is the $11,928/year individual rate minus $4,697, or $7,231." -us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed countable self-employment income exceeds the couple rate. That only holds if the one-half earned-income exclusion is skipped. With it, the head has $391.42/month of countable income against the $994 individual rate, for $7,231/year." -us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"The model used a projected $1,486 couple rate because it treated both spouses as eligible. Only the head is SSI-eligible, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"The model applied the $17,832 couple rate and did not carry the unused general exclusion over to earnings, producing $4,810 of countable income. The head alone qualifies at the $11,928 individual rate with $4,697 of countable income, for $7,231/year." -us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,832 couple rate, treating the spouse as eligible. The spouse is not SSI aged/blind/disabled, so the head receives the $11,928 individual rate minus $4,697, or $7,231." -us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model used a $17,835 couple rate and applied the $85/month of exclusions to earnings only. Only the head qualifies, so the $994/month individual rate applies, less $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"The model used a couple rate of about $1,490 and understated countable earnings at about $370/month. The head alone is eligible at the $994 individual rate with $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SSI value and no reasoning. The correct derivation pays the head alone the $994/month individual rate minus $391.42 of countable income, or $7,231/year." -us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model subtracted $4,709 from the $17,892 couple rate, treating both spouses as eligible. The spouse fails the SSI aged/blind/disabled test, so the head receives the $11,928 individual rate minus $4,697, or $7,231." -us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"The model took the small disability-benefit amounts as evidence that the household gets no SSI, ignoring the instruction to assume take-up. The blind, disabled head qualifies at $994/month minus $391.42 of countable income, for $7,231/year." -us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"The model used the $1,491 couple rate and applied the full $85 of exclusions to earnings without accounting for the head's $1.17 of unearned income. Only the head is eligible, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/year." -us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model cycled through invented rates between $674 and $1,058/month, treated the couple as jointly eligible, and submitted $8,472, which none of its own calculations produced. The correct individual rate is $994/month; less the head's $391.42 of countable income, that gives $7,231/year." -us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model asserted that the household's income and resources produce no benefit. The head meets the $2,000 resource limit and, after the $20, $65 and one-half exclusions, has $391.42/month of countable income against the $994 individual rate, for $7,231/year." +us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"Submitted a placeholder 0 without doing any SSI computation. The head is SSI-eligible through blindness with $912 in resources, under the $2,000 individual limit. Countable income of $391.42/month against the $994 individual FBR yields $602.58/month, or $7,231/yr." +us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated the 20-year-old spouse as an SSI-eligible disabled person and applied the $1,491/month eligible-couple FBR ($17,892/yr), pooling both spouses' $38 of disability benefits. Only the head is aged/blind/disabled, so the $994/month individual FBR applies against the head's own countable income of $391.42/month, which yields $7,231." +us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Treated the $14 and $24 disability benefits as monthly amounts and subtracted only the $65/month earned exclusion, never applying the one-half earned-income exclusion. It then declared ineligibility even though its own $10,076 countable income was below its own couple FBR. The correct head-only countable income is $391.42/month against the $994 individual FBR, which yields $7,231." +us,scenario_057,ssi,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Assumed no SSI without running the income test. The blind head passes the resource test, and after the $20 general exclusion, the $65 earned exclusion and halving, the $10,400 of self-employment income leaves countable income of $391.42/month. Against the $994/month individual FBR that leaves $602.58/month, or $7,231/yr." +us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Applied an approximately $17,604 couple FBR on the assumption that the spouse is SSI-eligible, and double-used the $20 general exclusion against both earned and pooled unearned income. Only the head is eligible, at the $994/month individual FBR, with countable income of $391.42/month. That yields $7,231." +us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Used an inflated $1,538/month couple FBR on the assumption that both spouses are SSI-eligible, then reported a number that does not follow from its own $14,271.50 arithmetic. The head alone is eligible at the $994/month individual FBR, less $391.42/month of countable income, which is $7,231/yr." +us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"Counted about $9,600 of net earnings as countable income and skipped the one-half earned-income exclusion, so it wrongly concluded income exceeds the FBR. After the $20 and $65 exclusions and halving, the head's countable income is $391.42/month, below the $994 individual FBR, which yields $7,231/yr." +us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the spouse as SSI-eligible, applied the $1,491 couple FBR and pooled both spouses' unearned income. The head is an eligible individual with an ineligible spouse, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/yr." +us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Applied a $1,486/month couple FBR on the assumption that both spouses qualify, and in its final pass dropped the unused general exclusion from earned income ($4,828 countable). Only the head is SSI aged/blind/disabled, so the $994 individual FBR less $391.42/month of countable income gives $7,231." +us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Computed a positive couple-rate benefit of about $12,590 using a $1,450 couple FBR, then submitted 0, contradicting its own work. The couple rate is also wrong because the spouse is not SSI-eligible. The head's individual benefit is $994 − $391.42 = $602.58/month, or $7,231/yr." +us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Treated both spouses as an eligible couple at the $1,491/month couple FBR. The spouse is not SSI aged/blind/disabled, so the head receives the $994/month individual FBR less $391.42 of countable income, which is $7,231/yr." +us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Paid the spouse a separate near-full individual benefit of about $991/month, but the spouse is not SSI aged/blind/disabled and receives $0. It also overstated the head's countable income at about $430/month instead of $391.42. The head alone receives $602.58/month, or $7,231/yr." +us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Correctly derived about $392 of countable income from pooled exclusions, but subtracted it from a $1,483 couple FBR on the assumption that the spouse is eligible. With only the head eligible, the $994 individual FBR less $391.42 yields $7,231/yr." +us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied the $1,486 eligible-couple FBR and overstated countable income at $418.83 by failing to carry the unused general exclusion over to earnings. The head alone is eligible at the $994 individual FBR, with $391.42/month of countable income, which gives $7,231." +us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR on the assumption that the spouse is SSI-eligible. Only the blind head qualifies, so the benefit is the $994 individual FBR less $391.42 of countable income, or $7,231/yr." +us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Applied a $1,486/month couple FBR for two disabled spouses and pooled their $38 of unearned income. The spouse is an ineligible spouse, so the head receives $994 − $391.42 = $602.58/month, or $7,231/yr." +us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Asserted that $10,400 of earnings disqualifies the household, without applying the $20 general exclusion, the $65 earned exclusion or the one-half earned-income exclusion. Those reduce the head's countable income to $391.42/month, below the $994 individual FBR, which yields $7,231/yr." +us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Used the couple maximum benefit on the assumption that both spouses are SSI-eligible. Only the head is aged/blind/disabled, so the $994/month individual FBR less $391.42 of countable income gives $7,231/yr." +us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Subtracted $4,709 of countable income from an approximately $17,400 couple FBR on the assumption that both spouses qualify. The head alone is eligible at the $994/month individual FBR with $4,697/yr of countable income, which yields $7,231." +us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Claimed that income and resources exceed SSI limits. Resources of $912 are under the $2,000 individual limit, and after the earned-income exclusions the head's countable income is $391.42/month, below the $994 FBR, which yields $7,231/yr." +us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Computed an eligible-couple benefit on the assumption that the 20-year-old spouse is SSI-disabled. The spouse is ineligible, so the head receives the $994 individual FBR less $391.42 of countable income, or $7,231/yr." +us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated both adults as an eligible couple and applied the about $1,486 couple FBR. Only the blind head is SSI aged/blind/disabled, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/yr." +us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Declared SSI not simulated and returned 0, ignoring the instruction to assume take-up. The blind head meets the resource and income tests and receives $602.58/month ($994 FBR − $391.42 countable income), or $7,231/yr." +us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Assumed that $10,400 of self-employment income exceeds SSI limits and treated the absence of listed SSI receipt as disqualifying, despite the take-up instruction. After the $20 and $65 exclusions and halving, countable income is $391.42/month, below the $994 individual FBR, which yields $7,231/yr." +us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"Subtracted $390.83 of countable income from an approximately $1,490.56 couple FBR, treating the spouse as SSI-eligible. The head alone is eligible at the $994 individual FBR, with $391.42 of countable income, which gives $7,231/yr." +us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed assets exceed the SSI limit, but $912 of bank assets is under the $2,000 individual resource limit, and earnings are mostly excluded. The head's countable income of $391.42/month against the $994 FBR yields $7,231/yr." +us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SSI as an unlisted receipt and set it to 0, instead of computing eligibility under the take-up assumption. The blind head qualifies, and $994 − $391.42 countable income gives $602.58/month, or $7,231/yr." +us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491 eligible-couple FBR ($17,892/yr) and pooled both spouses' disability benefits, on the assumption that the spouse is SSI-eligible. Only the head qualifies, so the $994 individual FBR less $4,697/yr of the head's countable income yields $7,231." +us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Used a couple FBR (about $1,450/month) on the assumption that both adults are SSI-disabled, and overstated countable income by not carrying the unused general exclusion to earnings. The head alone is eligible at $994/month less $391.42, which is $7,231/yr." +us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Reduced the 2026 couple maximum on the assumption that both spouses qualify. The spouse is not SSI aged/blind/disabled, so the head's individual FBR of $994/month less $391.42 of countable income gives $7,231/yr." +us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Computed an eligible-couple benefit of $13,183 from the $1,491 couple FBR. Only the blind head is eligible, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/yr." +us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Measured $4,709 of countable income against the $17,892 couple maximum, treating the spouse as SSI-eligible. The head is an eligible individual with an ineligible spouse: $11,928 individual FBR less $4,697 of the head's own countable income yields $7,231." +us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR on the assumption that both spouses are SSI-disabled. Only the head qualifies, at the $994 individual FBR, with $391.42/month of countable income, which yields $7,231/yr." +us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Reduced the $1,491/month couple benefit by countable earnings, treating the spouse as eligible. The spouse is not SSI aged/blind/disabled, so the $994 individual FBR less $391.42 gives $7,231/yr." +us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Applied the $17,892 couple maximum for two disabled spouses. Only the head is SSI-eligible, so the $11,928 individual FBR less $4,697 of countable income yields $7,231." +us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"Concluded that countable self-employment income exceeds the FBR, ignoring the one-half earned-income exclusion. After the $20 and $65 exclusions and halving, the head's countable income is $391.42/month, below the $994 individual FBR, which yields $7,231/yr." +us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"Used a projected $1,486 couple FBR on the assumption that both spouses are eligible. The head alone is eligible at the $994 individual FBR less $391.42 of countable income, which gives $7,231/yr." +us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"Applied a $17,832 couple FBR, and its $4,810 countable income wastes the general exclusion on unearned income. Only the blind head is eligible, with $4,697 of countable income against the $11,928 individual FBR, which yields $7,231." +us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Treated both spouses as an eligible couple at $1,486/month. The spouse is an ineligible spouse, so the head receives $994 − $391.42 = $602.58/month, or $7,231/yr." +us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Subtracted $4,690 of countable income from an approximately $17,835 couple FBR on the assumption that both spouses qualify. The head alone is eligible at the $11,928 individual FBR with $4,697 of countable income, which yields $7,231." +us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"Used an approximately $1,490 couple FBR and understated countable income at about $370/month. Only the head is SSI-eligible, so the $994 individual FBR less $391.42 of countable income gives $7,231/yr." +us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for ssi. The correct value is the head's individual benefit: $994 FBR less $391.42 of countable income, or $7,231/yr." +us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Applied the $17,892 eligible-couple rate on the assumption that both spouses are disabled for SSI. Only the head qualifies, so $11,928 less the head's $4,697 of countable income yields $7,231." +us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Inferred from the small disability-benefit amounts that no SSI is payable and never ran the SSI income test. The blind head passes the resource test, and countable income of $391.42/month against the $994 FBR yields $7,231/yr." +us,scenario_057,ssi,muse-wonder,prompt_ambiguity,age_disability,False,"Used the approximately $1,491 couple FBR with about $389 of countable income, treating the spouse as eligible. The head alone is eligible at $994/month less $391.42, which is $7,231/yr." +us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"Subtracted $390.83 of countable income from the $1,491 couple FBR, treating the spouse as SSI-disabled. Only the head qualifies, so the $994 individual FBR less $391.42 yields $7,231/yr." +us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Treated the annual $14 and $24 disability benefits as monthly amounts and cycled through invented FBRs ($674 to $1,058) before submitting $8,472, which matches none of its own computations. The 2026 individual FBR is $994/month and the head's countable income is $391.42/month, which gives $7,231/yr." +us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that income and resources produce no benefit without applying the SSI exclusions. Resources of $912 are under the limit, and after the $20 and $65 exclusions and halving, the head's countable income is $391.42/month, below the $994 FBR, which yields $7,231/yr." us,scenario_057,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied Louisiana's correct post-reform 3% flat rate but subtracted only $9,000 as the ""MFJ standard deduction"" — that is the repealed $4,500-per-filer personal exemption amount, not the deduction. Under Act 11 of the 2024 Third Extraordinary Session, Louisiana's standard deduction for married filing jointly is $25,000 (indexed beginning 2026), which exceeds the household's ~$9,665 federal AGI, so Louisiana taxable income is zero. Its $665 base is arithmetic on the wrong subtraction, and 3% of a base that does not exist produced the $20." us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It reconstructed the pre-2025 Louisiana regime on two counts: it subtracted $4,500-per-filer personal exemptions ($9,000) and applied a graduated first-bracket rate, both eliminated effective tax year 2025 when Louisiana moved to a 3% flat tax with a $25,000 MFJ standard deduction. Its cited ""1.75% on the first $12,500"" matches neither the current flat rate nor the former 1.85% first bracket, whose MFJ width was $25,000. The $25,000 standard deduction alone wipes out the ~$9,665 federal AGI, leaving $0 rather than a $665 taxable base." us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the miss is a contract failure rather than a substantive rule error. The required derivation runs $10,400 of SSTB self-employment income to roughly $9,665 federal AGI after the deductible half of self-employment tax, then against Louisiana's $25,000 married-filing-jointly standard deduction under its 3% flat tax, giving $0." @@ -5173,31 +5229,32 @@ us,scenario_060,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Th us,scenario_060,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model stated 'Veterans benefits are generally excluded from SNAP income' and dropped the entire $45,600, cutting countable gross income from $62,775 to $17,175. Veterans' benefits are explicitly counted unearned income under 7 CFR 273.9(b)(2)(ii), which lists 'annuities; pensions; retirement, veteran's or disability benefits' among income that must be counted. With the $45,600 restored, half of adjusted income (~$23,800/year) exceeds the $26,400 rent only slightly, leaving net income near $45,000/year (~$3,760/month) — far above the one-person net limit — instead of the $0 net income that produced the model's full $292/month maximum allotment." us,scenario_060,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model asserted 'VA disability benefits are excluded' and computed countable income from only Social Security, wages, and interest. VA compensation and pension payments are counted unearned income for SNAP under 7 CFR 273.9(b)(2)(ii); the narrow VA exclusions concern aid-and-attendance-type payments that reimburse specific medical or attendant costs, not the benefit stream itself. Excluding $45,600 let the uncapped elderly shelter deduction drive net income to $0 and triggered the one-person maximum allotment ($300/month), whereas counting it leaves net income around $3,760/month, above the 100% FPL net income limit, for a $0 benefit." us,scenario_060,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model set gross countable income at $1,431.25/month — wages, Social Security, and interest only — omitting the $45,600 in veterans benefits that 7 CFR 273.9(b)(2)(ii) requires be counted as unearned income. That omission also inflated the shelter deduction: with the true income counted, half of adjusted income is roughly $1,987/month rather than $90, so excess shelter is about $213/month instead of $2,109.63. Net income is then roughly $3,760/month, failing the one-person net income test, so the allotment is $0 rather than the $291/month maximum the model's $0 net income produced." -us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $28,800 survivor-benefits input as taxable income. That pushed provisional income to $55,786 and made 85% ($18,115) of Social Security taxable, for an AGI of $63,245 instead of $17,323. It also used qualifying-surviving-spouse status, which requires a dependent child. The correct AGI of $17,323 is fully covered by the $16,100 standard deduction, the $2,050 aged addition and the $6,000 senior deduction, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It included the $28,800 survivor benefits in gross income, which made $18,115 of Social Security taxable and raised AGI to $63,245 instead of $17,323. It also left out the $6,000 senior deduction. With AGI at $17,323, the single standard deduction plus the aged addition alone ($18,150) brings taxable income to zero." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $28,800 survivor benefits as taxable, reaching AGI of $63,245 with 85% of Social Security taxable. Excluding that input, provisional income is $26,986, only $993 of Social Security is taxable, and AGI is $17,323. It also omitted the $6,000 senior deduction; the deductions available exceed that AGI, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It computed the correct $17,323 AGI but applied TCJA-sunset law: an $8,250 standard deduction, a $2,000 aged addition and a $5,250 personal exemption. OBBBA made the TCJA structure permanent, giving a 2026 single deduction of $16,100 + $2,050 aged + $6,000 senior, which zeroes taxable income. Its submitted $157.30 also contradicts the $182.30 in its own explanation." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation lists survivor benefits as taxable income, so it taxed the $28,800 input and the resulting 85%-taxable Social Security, producing $5,135. AGI excluding that input is $17,323, and the $16,100 standard deduction plus the $2,050 aged addition and the $6,000 senior deduction eliminate taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $28,800 survivor benefits to AGI, giving $63,245 with $18,115 of taxable Social Security instead of $17,323 with $993 taxable. It then assumed TCJA had expired and used an $8,000 deduction, a personal exemption and a 15%/25% bracket schedule. Under 2026 OBBBA law, the $16,100 + $2,050 + $6,000 deductions exceed the correct AGI, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It put the full $21,312 of Social Security and the $28,800 survivor benefits into AGI and subtracted the farm rent instead of adding it, reaching $46,649. It then reported $178 of tax on $29,449 of taxable income, which does not follow from any bracket schedule. The correct AGI is $17,323 ($4,793 + $257 + $11,280 + $993 taxable Social Security), and that is below the $18,150 standard-plus-aged deduction." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It got AGI right at $17,323 but assumed TCJA expired in 2026, restoring a personal exemption and a pre-TCJA standard deduction that together total about $15,800. OBBBA kept the higher deduction: $16,100 single + $2,050 aged + the $6,000 senior deduction, or $24,150, which exceeds AGI and leaves $0 tax." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $28,800 survivor benefits and 85% of Social Security, giving AGI of $63,245 instead of $17,323. It also applied TCJA-sunset deductions and exemptions of about $15,400. The correct AGI against the $24,150 of 2026 OBBBA deductions yields zero taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It included the $28,800 survivor benefits in gross income, which made $18,115 of Social Security taxable, for an AGI of $63,245. It also used TCJA-sunset rules: an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. Without that input, AGI is $17,323, and the OBBBA $16,100 + $2,050 + $6,000 deductions wipe out taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It taxed the survivor benefits and 85% of Social Security and applied a post-TCJA-sunset standard deduction plus personal exemption, producing $6,546.75. AGI excluding the $28,800 input is $17,323, and the 2026 single standard deduction ($16,100), aged addition ($2,050) and senior deduction ($6,000) reduce taxable income to zero." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It built AGI of $63,245 by including the $28,800 survivor benefits and $18,115 of taxable Social Security. Excluding that input, provisional income ($26,986) is just above the $25,000 single base, only $993 of Social Security is taxable, and AGI of $17,323 falls entirely within the standard, aged and senior deductions." -us,scenario_062,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly found taxable income below zero but then subtracted $1,031 of nonrefundable elderly/disabled credit to report a negative tax. Nonrefundable credits are capped at tax liability. The Schedule R credit is also $0 here because $20,319 of nontaxable Social Security exceeds the $5,000 initial amount. Tax before refundable credits is therefore $0, not -$1,031." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $28,800 survivor benefits as taxable. That drove 85% of Social Security into income and produced tax of $4,328 even after the senior and QBI deductions. Without that input, AGI is $17,323, which is below the $18,150 standard-plus-aged deduction before the senior deduction is even applied." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It included 'taxable survivor' income of $28,800 in AGI, which pushed provisional income past the $34,000 single threshold and made $18,115 of Social Security taxable. The correct AGI is $17,323 with only $993 of Social Security taxable, and the single standard, aged and senior deductions reduce it to zero taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reached $18,115 of taxable Social Security only by adding the $28,800 survivor-benefits input to gross income. Excluding it, taxable Social Security is $993 and AGI is $17,323, which the $16,100 standard deduction plus the $2,050 aged addition and the $6,000 senior deduction fully offset, so there is no tax and the $959 QBI deduction is moot." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly treated the $28,800 survivor payment as taxable non-Social-Security income, which inflated AGI to $63,245.20 and made 85% of Social Security taxable. With that input excluded, AGI is $17,323 against its own $24,150 of standard, aged and senior deductions, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse even though that status requires a dependent child; with no dependents, the head files single. It also taxed the $28,800 survivor benefits, which made $18,115 of Social Security taxable. The correct single AGI is $17,323, which the $16,100 + $2,050 + $6,000 deductions reduce to zero taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It included the $28,800 survivor benefits and 85% of Social Security in AGI ($63,245). It also applied TCJA-sunset rules: a roughly $16,049 standard deduction plus exemption, with 10%/15% brackets. OBBBA made the TCJA structure permanent, and the correct $17,323 AGI is fully offset by the $16,100 standard deduction, $2,050 aged addition and $6,000 senior deduction." -us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted $45,130 of 'other taxable income' by including the $28,800 survivor benefits, making $18,115 of Social Security taxable. It then used post-TCJA-sunset deductions ($10,612 plus a $5,442 exemption) and a 15% bracket. The correct AGI of $17,323 against the 2026 OBBBA deductions of $24,150 gives $0." -us,scenario_062,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $28,800 survivor benefits to AGI, giving $63,245 with 85% of Social Security taxable, and it also omitted the $6,000 senior deduction. Excluding that input, AGI is $17,323, which is already below its own $18,150 single-plus-aged standard deduction, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so the answer could not be scored." -us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It computed the correct $17,323 AGI with $993 of taxable Social Security but applied a 'restored exemption' and pre-TCJA standard deduction, leaving about $1,820 taxable. OBBBA kept the TCJA structure: the $16,100 single deduction, the $2,050 aged addition and the $6,000 senior deduction exceed AGI, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It labeled the $28,800 survivor benefits as taxable and added them to gross income, which made 85% ($18,115) of Social Security taxable and produced AGI of $63,245. Without that input, AGI is $17,323, and its own $23,750 of standard, aged and senior deductions eliminates taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It settled on taxing the $28,800 survivor benefits as SBP-type income and 85% of Social Security, for AGI of $63,245. It also used a $16,550 deduction and omitted the $6,000 senior deduction. The correct AGI is $17,323, and the 2026 single standard deduction ($16,100) plus the aged addition ($2,050) and the senior deduction reduce it to zero taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Added the $28,800 survivor benefits to AGI and provisional income, which made 85% of SS taxable ($18,115) and produced AGI of $63,245. Without that amount, AGI is about $17,323 (under $16,330 with the joint thresholds the model used). That is below even the single deductions of $24,150 (standard deduction plus aged add-on plus senior deduction), so taxable income is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Counted the $28,800 survivor benefits as taxable income, which inflated provisional income and made 85% of SS taxable. It also left out the 2026 $6,000 senior deduction. Correct AGI is about $17,323, which is below the $24,150 of 2026 deductions, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Included the $28,800 survivor benefits in AGI, which pushed taxable SS to $18,115 and AGI to $63,245. It also omitted the $6,000 senior deduction and used outdated bracket widths. Correct AGI of about $17,323 is fully absorbed by the $16,100 standard deduction, the $2,050 aged add-on and the $6,000 senior deduction." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly excluded the survivor benefits and reached AGI of $17,323, but assumed the TCJA had expired in 2026. It applied an $8,250 standard deduction plus an exemption instead of the OBBBA $16,100 standard deduction, $2,050 aged add-on and $6,000 senior deduction, which together exceed AGI and give $0. Its submitted $157.30 also contradicts the $182.30 in its own reasoning." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the $28,800 survivor benefits as taxable income alongside 85% of SS, which gave a large positive taxable income. Correct AGI is only about $17,323, and the 2026 standard, aged and senior deductions reduce taxable income to $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Added the $28,800 survivor benefits to AGI, which made 85% of SS taxable. It then assumed the TCJA had sunset, applying pre-TCJA deductions and 15%/25% brackets. Under 2026 OBBBA law, AGI of about $17,323 is below the $24,150 of deductions, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Built AGI by adding the full $21,312 SS and the $28,800 survivor benefits while dropping the farm rent. It applied no provisional-income test, and its $178 does not follow from its own $29,449 of taxable income. Correct AGI is about $17,323, which the 2026 deductions fully offset." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly reached AGI of about $17,323 but assumed the TCJA expired in 2026, using roughly $15,800 of pre-TCJA deductions and exemptions. OBBBA made the higher standard deduction permanent ($16,100 plus a $2,050 aged add-on) and added a $6,000 senior deduction, which leaves taxable income at $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted the $28,800 survivor benefits as taxable and taxed 85% of SS, which gave AGI of $63,245. It then applied pre-TCJA deductions of about $15,400. Correct AGI is about $17,323, which is below the 2026 OBBBA deductions of $24,150, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Included the $28,800 survivor benefits in gross income, which made $18,115 of SS taxable. It also used TCJA-sunset deductions and exemptions with 15%/25% brackets. With correct AGI of about $17,323 and the 2026 standard, aged and senior deductions, taxable income is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Taxed the survivor benefits along with 85% of SS, and applied a restored personal exemption under the assumption that the TCJA had sunset. Correct AGI is about $17,323, and the $24,150 of 2026 deductions eliminates taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Used AGI of $63,245, which adds the $28,800 survivor benefits and 85% of SS. Excluding the survivor benefits gives provisional income of $26,986, so only $993 of SS is taxable and AGI is about $17,323. That AGI falls below the 2026 standard, aged and senior deductions, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly found zero taxable income, then subtracted $1,031 of nonrefundable credits to produce a negative liability. Nonrefundable credits are capped at pre-credit tax, so with $0 tax before credits the result is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Included the $28,800 survivor benefits in taxable income, which made 85% of SS taxable and produced positive taxable income even after the senior and QBI deductions. Correct AGI is about $17,323, which the 2026 deductions fully offset, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the $28,800 survivor benefits as taxable income, which raised provisional income enough to tax 85% of SS. Without it, AGI is about $17,323, which is below the $16,100 standard deduction plus the $2,050 aged add-on and the $6,000 senior deduction." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed $18,115 of taxable SS by counting the $28,800 survivor benefits in provisional income and AGI. Correct AGI is about $17,323, with only $993 of SS taxable, and the 2026 deductions reduce taxable income to $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the correct 2026 deductions ($16,100 standard, $2,050 aged and $6,000 senior) but treated the $28,800 survivor payment as taxable income, which inflated AGI to $63,245. Excluding it gives AGI of about $17,323, which is below $24,150 of deductions, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the $28,800 non-SS survivor benefits as taxable, which made 85% of SS taxable and gave AGI of $63,245. Without it, provisional income is about $26,986, which is below the $32,000 joint threshold the model used, so AGI is $16,330 and its $39,850 of deductions zero out tax." +us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Included the survivor benefits and 85% of SS in AGI ($63,245), then applied TCJA-sunset exemptions and brackets. Correct AGI is about $17,323, which the 2026 OBBBA standard, aged and senior deductions fully offset." +us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Added $45,130 of other income, including the $28,800 survivor benefits, to provisional income, which made $18,115 of SS taxable. It also used post-TCJA-sunset deductions and 15% brackets. Correct AGI is about $17,323, which is below the 2026 deductions of $24,150." +us,scenario_062,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Counted the $28,800 survivor benefits in AGI, which made 85% of SS taxable, and omitted the $6,000 senior deduction. Correct AGI is about $17,323, and the $18,150 single-plus-aged standard deduction alone exceeds it, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or reasoning for this output. The correct derivation has AGI of about $17,323 fully offset by the 2026 standard, aged and senior deductions, for $0 of tax." +us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly computed AGI of $17,323 but applied a restored personal exemption and pre-TCJA standard deduction under a TCJA-sunset assumption, leaving about $1,820 taxable. Under 2026 OBBBA law, the $16,100 standard deduction, $2,050 aged add-on and $6,000 senior deduction exceed AGI, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Included the $28,800 survivor income and 85% of SS in AGI ($63,245) and omitted the $6,000 senior deduction. Excluding the survivor benefits gives AGI of about $16,330 with joint thresholds, which its own $33,850 surviving-spouse deduction already eliminates, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the $28,800 survivor benefits as taxable gross income, which made $18,115 of SS taxable and AGI $63,245. With correct AGI of about $17,323, its own $16,100 standard deduction plus the aged add-on and the $6,000 senior deduction reduce taxable income to $0." +us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Added the $28,800 survivor benefits to income and taxed 85% of SS, giving AGI of $63,245. It also left out the $6,000 senior deduction and used a mix of 2024 and 2025 standard-deduction figures. Correct AGI is about $17,323, which the 2026 deductions fully offset, so tax is $0." us,scenario_062,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_062,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It computed $32,849 of countable income against the ~$15,060 poverty guideline and correctly flagged that $29,100 in bank assets blows through the $2,000 SSI-related resource limit, then discarded both of its own disqualifying findings by invoking Florida's medically needy share-of-cost program and spousal protections. The aged/disabled route to Medicaid attaches through SSI receipt, and SSI here is $0, so medicaid_category is NONE; medically needy spend-down confers no eligibility. It also omitted the $4,793 of farm rent from its countable total, understating income by that amount." us,scenario_062,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"It struck the $21,312 of Social Security survivor benefits from countable income as ""largely excluded,"" but Social Security is counted in full as unearned income under SSI-related rules past the $20 general disregard, raising countable income from its stated $16,330 to about $37,600 — roughly three times the ~$12,000 annual SSI federal benefit rate — and it never applied the $2,000 resource limit to $29,100 of bank assets. Because SSI is $0, no SSI-linked aged/disabled category attaches and medicaid_category is NONE, so its ""low countable income relative to the aged/disabled standard"" premise fails on both the income and resource tests." @@ -5234,37 +5291,38 @@ us,scenario_064,child1_wic_eligible,glm-5.3,parse_contract_failure,missing_outpu us,scenario_064,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that a 12-year-old is ""in the WIC-eligible age range,"" but WIC's child category ends at the fifth birthday, so a 12-year-old has no WIC categorical status at all — the model appears to have imported a school-age or free-school-meals age frame into WIC. It also skipped the WIC income test entirely, which caps countable income at 185% of the federal poverty guideline and disqualifies this household's ~$98,810 of income for 5 people regardless of age." us,scenario_064,dependent1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the head's and spouse's tax-unit income toward dependent1's Medicaid eligibility and compared it to roughly $32k (100% FPL for 5). The engine sets dependent1's MAGI income level at 0.00× FPL, since this 27-year-old adult has no income, so they pass the Wisconsin adult limit easily and are eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility because the household's bank, stock and home assets exceed Medicaid asset limits. Dependent1 qualifies under the MAGI adult category, which has no asset test, and their MAGI income level is 0.00× FPL, so the assets are irrelevant and dependent1 is eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model judged dependent1 on household income against the disabled-group limit. The engine uses the MAGI adult category, where dependent1's own income level is 0.00× FPL, so dependent1 is eligible no matter how much the parents earn." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model attributed the head's roughly $97k wages, 401(k) distributions and partnership income to dependent1's MAGI test. The engine evaluates this 27-year-old adult dependent at 0.00× FPL, which is below Wisconsin's adult threshold, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model measured dependent1 against total household income. The engine's MAGI adult test counts dependent1's own zero income (0.00× FPL), so dependent1 meets the Wisconsin adult income limit and is eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model assumed household income counts toward this adult dependent. The engine computes dependent1's MAGI income level as 0.00× FPL under the adult category, which makes them Medicaid-eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model added up the head's wages, partnership, farm and 401(k) income as dependent1's household MAGI and found it above 138% FPL. The engine puts dependent1's MAGI at 0.00× FPL in the adult category, so they are below the threshold and eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied household income to this adult dependent. The engine's MAGI adult test uses dependent1's income level of 0.00× FPL, which qualifies them for Medicaid." -us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly identified Wisconsin's 100% FPL adult limit but compared it to household income. Dependent1's MAGI income level in the engine is 0.00× FPL, which is under that limit, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the household's income as exceeding dependent1's Medicaid limit. The engine evaluates dependent1 in the MAGI adult category at 0.00× FPL because they have no income, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reason for its 'not eligible' answer, which matches charging the parents' income to dependent1. The correct derivation places this 27-year-old with no income in the MAGI adult category at 0.00× FPL, which makes them eligible." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied household income to dependent1's adult Medicaid test. The engine computes dependent1's MAGI income level as 0.00× FPL, below the Wisconsin adult threshold, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility based on household income. The engine's adult-category test uses dependent1's MAGI income level of 0.00× FPL, which qualifies them." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model compared family income to the Wisconsin adult limit. The engine measures dependent1's MAGI at 0.00× FPL, so dependent1 falls under the adult limit and is eligible." -us,scenario_064,dependent1_medicaid_eligible,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the head's $100k+ tax-unit income as dependent1's MAGI and treated the employer-sponsored insurance as disqualifying. The engine puts dependent1's MAGI income level at 0.00× FPL in the adult category, and having ESI does not bar Medicaid eligibility, so dependent1 is eligible." -us,scenario_064,dependent1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for dependent1_medicaid_eligible. The correct answer is eligible (1), because dependent1 is in the MAGI adult category at 0.00× FPL." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility because the household is high-income. The engine evaluates this adult dependent on their own MAGI income level of 0.00× FPL, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model defaulted to 'not eligible' because of disability-pathway details and high household income. The relevant pathway is the MAGI adult category, which needs no disability details. Dependent1's income level there is 0.00× FPL, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the family's household MAGI to dependent1 because they are a tax dependent. The engine sets dependent1's MAGI income level at 0.00× FPL for the adult category, which is below Wisconsin's limit, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model invoked income and resource rules to deny eligibility. The MAGI adult category has no resource test, and dependent1's MAGI income level is 0.00× FPL, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model relied on household income and assets. Dependent1 qualifies through the MAGI adult category, which has no asset test, at a MAGI income level of 0.00× FPL, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model said income exceeds the thresholds, using household income. Dependent1's own MAGI income level in the engine is 0.00× FPL, which is below the adult limit, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model said dependent1 fails both the ABD $2,000 resource test and the MAGI adult limit. The MAGI adult category has no resource test, and dependent1's MAGI income level is 0.00× FPL, so they pass the adult pathway and are eligible." -us,scenario_064,dependent1_medicaid_eligible,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly chose the MAGI adult rules but counted household income against them, and added an irrelevant ABD resource test. The engine computes dependent1's adult-category MAGI at 0.00× FPL, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated household MAGI as dependent1's income for the adult test. The engine sets dependent1's MAGI income level at 0.00× FPL, so they are eligible under the adult category even without SSI." -us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave dependent1 the household's roughly 273% FPL MAGI and also applied ABD asset limits. The engine evaluates dependent1 in the MAGI adult category at 0.00× FPL with no asset test, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for dependent1_medicaid_eligible. The correct answer is eligible (1), because dependent1 is in the MAGI adult category at 0.00× FPL." -us,scenario_064,dependent1_medicaid_eligible,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reasoning, and its 'not eligible' answer matches charging the parents' income to dependent1. The correct derivation evaluates this 27-year-old with no income in the MAGI adult category at 0.00× FPL, which makes them eligible." -us,scenario_064,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave dependent1 the family's roughly 350% FPL MAGI and also applied SSI-related asset tests. The engine measures dependent1's MAGI income level at 0.00× FPL in the adult category, which has no asset test, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly identified Wisconsin's roughly 100% FPL adult limit but counted the head's wages and partnership income as dependent1's MAGI. Dependent1's MAGI income level is 0.00× FPL, which is below that limit, so they are eligible." -us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model said dependent1's income is too high, but dependent1 has no income. Their MAGI income level is 0.00× FPL, so they qualify under the adult category." +us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave the claimed dependent the head and spouse's tax-unit MAGI and compared it with a 5-person 100% FPL limit. The engine measures Dependent 1's MAGI-based adult-category income level from their own zero income (0.00 x FPL), which passes the adult income test." +us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the head's bank, stock, and home assets against Dependent 1 under an asset test. Dependent 1 qualifies through the MAGI-based adult category, which has no asset test, and their own income level is 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model used household income and a disability-group limit. It never checked the MAGI adult category, where Dependent 1's own income is zero (0.00 x FPL) and passes the income test." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the head's high household MAGI to the 27-year-old dependent. The engine computes Dependent 1's adult-category MAGI level from their own zero income, 0.00 x FPL, which is below the threshold." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model tested Dependent 1 against the head's household income. The engine places them in the MAGI adult category with their own income at 0.00 x FPL, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave the adult dependent the household's income. Dependent 1's own MAGI income level is 0.00 x FPL, which qualifies them under the adult expansion category." +us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model compared the whole household's MAGI (head's wages, partnership income, and 401(k) distributions) with 138% FPL. The engine computes Dependent 1's adult-category income level from their own zero income (0.00 x FPL), which is eligible." +us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,The model applied household income to the adult dependent. The engine evaluates Dependent 1 under the MAGI adult category at their own 0.00 x FPL income level and finds them eligible. +us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly identified the adult BadgerCare category but tested the household's income against 100% FPL. Dependent 1's own MAGI is zero (0.00 x FPL), which is under the adult limit." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the household's income as Dependent 1's. Under the engine's MAGI adult category, Dependent 1's own income level is 0.00 x FPL, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reasoning, and its 0 implies it attributed the head's income to Dependent 1. The engine places Dependent 1 in the MAGI adult category at their own 0.00 x FPL income, which makes them eligible." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model tested household income against the WI adult threshold. The engine uses Dependent 1's own zero MAGI (0.00 x FPL), which is under the adult limit." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied eligibility on household income. Dependent 1's own MAGI income level is 0.00 x FPL, which qualifies them for the adult expansion category." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly used the adult category but applied family income to it. The engine measures Dependent 1's income level from their own zero income, 0.00 x FPL, which is eligible." +us,scenario_064,dependent1_medicaid_eligible,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model assigned the tax unit's $100k+ MAGI to the dependent and also cited employer-sponsored insurance as a barrier. The engine computes Dependent 1's MAGI level from their own zero income (0.00 x FPL), and ESI does not block Medicaid eligibility." +us,scenario_064,dependent1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No answer or explanation was returned for this output, so nothing could be scored. The correct result is eligible under the MAGI adult category at 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model judged eligibility from the household's high income. The engine places Dependent 1 in the adult category using their own 0.00 x FPL income level, which is eligible." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model defaulted to ineligible on simplified household-income assumptions. Dependent 1's own MAGI is zero (0.00 x FPL), which passes the adult expansion income test." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model explicitly applied the family's household MAGI to the adult tax dependent. The engine computes Dependent 1's adult-category income level from their own zero income, 0.00 x FPL, which is eligible." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied household income and resource rules. Dependent 1 qualifies through the MAGI adult category, which has no resource test, with their own income at 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model cited household income and assets. The MAGI adult category has no asset test, and Dependent 1's own income level is 0.00 x FPL, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the household's income to Dependent 1. The engine evaluates the adult category with Dependent 1's own zero MAGI (0.00 x FPL), which is below the threshold." +us,scenario_064,dependent1_medicaid_eligible,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied an ABD resource test and the household's MAGI. The engine qualifies Dependent 1 through the MAGI adult category, which has no asset test, at their own 0.00 x FPL income." +us,scenario_064,dependent1_medicaid_eligible,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly picked the MAGI adult rules but counted household income and resources against Dependent 1. The engine measures their own income at 0.00 x FPL, which passes the adult limit." +us,scenario_064,dependent1_medicaid_eligible,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model used household MAGI for the adult category. Dependent 1's own MAGI income level is 0.00 x FPL, which qualifies them without SSI receipt." +us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave the dependent a household MAGI of about 273% FPL and applied ABD asset limits. The engine evaluates the MAGI adult category, which has no asset test, at Dependent 1's own 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for this output, so nothing could be scored. The correct result is eligible under the MAGI adult category at 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reasoning, and its 0 implies it attributed household income to Dependent 1. The engine computes Dependent 1's adult-category MAGI at their own 0.00 x FPL, which makes them eligible." +us,scenario_064,dependent1_medicaid_eligible,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied household income and financial rules. Dependent 1's own MAGI is zero (0.00 x FPL), which qualifies them under the adult expansion category." +us,scenario_064,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave Dependent 1 a family MAGI of about 350% FPL and counted the head's bank and stock assets. The engine evaluates the MAGI adult category at Dependent 1's own 0.00 x FPL with no asset test, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the head's $97,295 wages and other income in Dependent 1's MAGI. The engine uses Dependent 1's own zero income (0.00 x FPL), which is under the adult limit." +us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model said Dependent 1's income is too high, but Dependent 1 has no income. Their own MAGI income level is 0.00 x FPL, which qualifies them for the adult expansion category." us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model assumed 'SSDI-based eligibility' just because Dependent 1 is disabled, but the household lists no SSDI benefits, and unlisted amounts are 0. The under-65 disability path needs 24 months of actual SSDI entitlement, which Dependent 1 does not have, so the correct answer is not eligible (0)." us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model said a disability alone qualifies someone for Medicare at any age. Under 65, Medicare instead requires 24 months of SSDI entitlement, or ESRD or ALS. Dependent 1 has no SSDI income, so a 27-year-old is not eligible." us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the 24-month SSDI entitlement rule but then applied it without any SSDI receipt, using only the disability flag. The household has $0 in Social Security disability benefits and no months of receipt, so the SSDI path does not apply. A 27-year-old with no ESRD or ALS is not eligible." @@ -5290,50 +5348,51 @@ us,scenario_064,dependent2_medicaid_eligible,grok-4.5,prompt_ambiguity,household us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for dependent2_medicaid_eligible, so its answer is missing rather than wrong on substance. The correct answer is 0: household MAGI of 3.22× FPL exceeds Wisconsin's child limits, and no disability, pregnancy or caretaker pathway applies." us,scenario_064,dependent2_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent2_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own chain reached about $67,713 taxable income (it used a $32,300 standard deduction instead of $32,200), about $7,630 of tax and $3,200 of CTC/ODC, which yields about $4,430. It then added an unexplained ~$1,529 of ""adjustments"" to get $5,959, so the final number does not follow from its own computation." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp income, but the $14,000 is already a component of that total. This inflated net business income to $16,550 and the QBI deduction to $3,310; the correct figures are $2,550 and $510. It also deducted the $1,796 domestic production deduction, which was repealed after 2017, leaving $77,217 taxable income instead of $67,812." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $13,899 overtime premium on top of wages that already include it and deducted the full $18,235 capital loss instead of the $3,000 limit. It used a $14,600 standard deduction instead of the 2026 MFJ $32,200, never took the OBBBA overtime deduction, and used a $2,000 CTC instead of $2,200 with no $1,000 ODC. It then applied an unexplained further reduction to reach $9,631." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no derivation and answered $0. The correct computation leaves $67,812.12 of taxable income after the $32,200 standard deduction, the $13,899 overtime deduction and the $510 QBI deduction, which produces $7,641.46 of tax. Only $3,200 of CTC/ODC offsets that, so the answer wrongly assumed deductions and credits wipe out all liability." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never took the 2026 OBBBA $13,899 qualified overtime deduction. It also computed SE tax on the $14,000 partnership income without netting the $16,800 farm loss (net SE earnings are negative, so there is no SE tax or half-SE deduction). It invented a split that made most of the $2,000 CTC refundable when the full $2,200 is nonrefundable here, then added $1,000 of unexplained ""adjustments""." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction. It also computed SE tax on $14,000 without netting the $16,800 farm loss and then added that SE tax into income tax, even though SE tax is not part of this measure. Finally it increased the total by unexplained amounts instead of subtracting the $3,200 CTC/ODC, ending at $14,959." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction and deducted the repealed $1,796 DPAD. It took a half-SE-tax deduction even though the farm loss makes net SE earnings negative, and used a ~$3,900 QBI deduction instead of $510. It then added an unspecified ""additional tax on other income"" and applied only $2,500 of credits instead of $3,200." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the 2026 OBBBA $13,899 overtime deduction and used the 2025 $30,000 standard deduction instead of $32,200. It also took a $1,560 SE health insurance deduction and a $989 half-SE-tax deduction even though netting the farm loss leaves no SE earnings, and used a $2,000 CTC instead of $2,200." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction, deducted the repealed $1,796 DPAD and took a half-SE-tax deduction even though net SE earnings are negative. It then treated the CTC as negligible and inflated the result to $12,500 for nonexistent Medicare/NIIT effects. The correct result subtracts the full $3,200 of CTC/ODC from $7,641.46 of tax." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp total, which pushed AGI to ~$128.5k instead of $114,421 and QBI to ~$3.3k instead of $510. It also omitted the $13,899 overtime deduction, so its taxable income (~$92.9k) far exceeds $67,812." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income (net business income $16,550 instead of $2,550), deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200 and omitted the $13,899 overtime deduction. It also claimed $4,000 CTC for two qualifying children, treating the 18-year-old as an under-17 child; the correct credits are $2,200 CTC plus two $500 ODCs." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law: a $15,900 standard deduction, $26,000 of personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent for 2026, so the correct rules are the $32,200 standard deduction, the 10%/12% brackets, a $2,200 CTC and $500 ODCs. It also subtracted health premiums from wages and omitted the $13,899 overtime deduction." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 law, with personal exemptions and a CTC phaseout starting at $110,000, which left only $150 of credit. Under OBBBA, 2026 has no personal exemptions, a $32,200 standard deduction, a $13,899 overtime deduction and a $400,000 MFJ phaseout threshold, so the full $3,200 of CTC/ODC applies." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It reached the correct $114,422 AGI and $510 QBI deduction but omitted the 2026 $13,899 qualified overtime deduction. It also deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200 and used a $2,000 CTC instead of $2,200, leaving $81,366 of taxable income instead of $67,812." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA rules: $25,250 of personal exemptions and a $1,000 CTC with no ODC. It also subtracted the $19,648 employer-sponsored premiums from the given wages and itemized medical expenses that included those employer premiums, and it treated the repealed $1,796 DPAD as a QBI deduction. It never applied the $32,200 standard deduction or the $13,899 overtime deduction." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no real derivation, and its $11,090 exceeds even the $7,641.46 pre-credit tax on the correct $67,812 taxable income. That means it overstated taxable income by leaving out the $13,899 overtime deduction and the correct $32,200 standard deduction, and it did not subtract the $3,200 CTC/ODC." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly used pre-TCJA 2026 rules: ~$26,000 of personal exemptions, a $16,200 standard deduction and a $1,000 CTC. OBBBA's permanent $32,200 standard deduction, $13,899 overtime deduction, $2,200 CTC and $500 ODCs apply instead. It also subtracted ESI premiums from wages, deducted the repealed DPAD and added a $2,000 early-withdrawal penalty that is not part of this regular-tax-less-nonrefundable-credits measure." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used personal exemptions and a $1,000 Child Tax Credit, which are pre-TCJA sunset rules, instead of 2026 OBBBA law. It also understated AGI at $94,756 instead of $114,421, omitted the $13,899 overtime deduction and the two $500 other-dependent credits, and did not use the $2,200 CTC." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation, and its $11,940 is above the $7,641.46 tax on the correct $67,812 taxable income even before credits. The answer therefore ignores the $13,899 overtime deduction and the $3,200 of CTC/ODC that reduce tax to $4,441.46." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $6,215 plus $3,200 of credits implies about $9,415 of pre-credit tax, or about $81,000 of taxable income. That matches the reference base without the 2026 $13,899 qualified overtime deduction, which it never mentions." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $4,324 plus $3,200 of credits implies about $7,524 of pre-credit tax, about $117 below the $7,641.46 due on $67,812.12. That means it understated taxable income by roughly $975 by taking deductions beyond the $32,200 standard, $13,899 overtime and $510 QBI deductions." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $4,057 plus $3,200 of credits implies about $7,257 of pre-credit tax, about $384 below $7,641.46, so taxable income came out roughly $3,200 too low. That gap matches overstated deductions: a QBI deduction on un-netted partnership income instead of 20% of the $2,550 net after the farm loss, or the repealed $1,796 DPAD." -us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It assumed deductions and credits fully offset the tax without computing anything. After the $32,200 standard deduction, $13,899 overtime deduction and $510 QBI deduction, $67,812 of taxable income remains, which produces $7,641.46 of tax, and only $3,200 of CTC/ODC offsets it." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It treated liability as zero without any computation. Taxable income of $67,812 after all applicable deductions produces $7,641.46 of tax, which exceeds the $3,200 of nonrefundable CTC/ODC by $4,441.46." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,219 plus $3,200 of credits implies only ~$5,419 of tax, or ~$49,300 of taxable income. That is about $18,500 below the correct $67,812, so it took exclusions beyond the $2,315 401(k) deferral, $3,000 capital-loss limit, $108 IRA, $32,200 standard, $13,899 overtime and $510 QBI deductions; one example is treating the $19,648 employer-paid ESI premium as a reduction of reported wages." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,485 implies about $7,685 of pre-credit tax, so its taxable income was roughly $365 above $67,812.12. It understated a deduction relative to the $510 QBI deduction (20% of $19,350 partnership income net of the $16,800 farm loss) and the $108 IRA deduction." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It subtracted only the $2,200 qualifying-child credit from roughly the correct ~$7,650 of tax. It omitted the $500 credit for other dependents for each of the disabled 27-year-old and the 18-year-old, which together take another $1,000 off." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its $92,978 AGI equals the correct $114,421 minus the $19,648 employer-paid ESI premium and the $1,796 DPAD. The given wages are not reduced by employer premiums, and the DPAD was repealed after 2017. This cut taxable income to $46,369 instead of $67,812." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $5,570 implies $77,217 of taxable income, which is $9,405 above the correct figure. That gap comes from adding the $14,000 SE partnership income on top of the $19,350 partnership total that already includes it, partly offset by deducting the repealed $1,796 DPAD and taking a $3,310 QBI deduction instead of $510." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $112,626 AGI is the correct $114,421 minus the repealed $1,796 DPAD. It also claimed $1,500 of other-dependent credits for three dependents when only the 27-year-old and the 18-year-old qualify, which is $1,000." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no derivation, and $13,500 is far above the $7,641.46 tax on $67,812 of taxable income. It ignored the $13,899 overtime deduction and the $3,200 of CTC/ODC that bring liability to $4,441.46." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It reached the correct $114,422 AGI but applied TCJA-sunset rules: a $17,000 standard deduction, $27,000 of personal exemptions, a 15% bracket and a $1,000 CTC phased out above $110,000. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction, a $13,899 overtime deduction, 10%/12% brackets, a $2,200 CTC and two $500 ODCs with a $400,000 phaseout threshold." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership total that already includes it, which inflated QBI to $16,550 (a $3,310 deduction instead of $510). It also used a $30,700 standard deduction instead of $32,200, giving $80,513 of taxable income instead of $67,812." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income, which is already inside the $19,350 partnership total, and took a $3,310 QBI deduction instead of $510, overstating taxable income at $79,013. It then added a $2,000 early-distribution penalty, which is not part of this income-tax-after-nonrefundable-credits measure." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 of employer and own health premiums from wages, double-counted the $14,000 SE partnership income, took a half-SE-tax deduction despite negative net SE earnings, and treated the repealed $1,796 DPAD as QBI. It also omitted the $13,899 overtime deduction and the $1,000 of ODC, and used a $2,000 CTC instead of $2,200." -us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"It added a $2,000 10% early-withdrawal penalty on the 401(k) distribution, which is not included in this measure. It also understated AGI at $97,723 and deducted the repealed $1,796 DPAD instead of the $510 QBI deduction. The correct result is $7,641.46 of tax on $67,812 minus $3,200 of credits." -us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income as separate from the $19,350 partnership total, took QBI on $33,350 without netting the $16,800 farm loss ($6,670 instead of $510), and deducted the dependent's own 401(k) and IRA contributions on the parents' return. It also omitted the $13,899 overtime deduction and denied the $500 ODC to the 18-year-old. Its final $7,646 is an unexplained adjustment from its own $7,815." -us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It did not exclude the head's $2,315 traditional 401(k) deferral from wages (AGI $116,737 instead of $114,421). It also deducted the repealed $1,796 DPAD in place of the $510 QBI deduction (20% of $19,350 − $16,800), which overstated taxable income by about $1,030." -us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It answered $0 with no reasoning. Taxable income of $67,812 after the standard, overtime and QBI deductions produces $7,641.46 of tax, and the $3,200 of CTC/ODC leaves $4,441.46." -us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership total, which created $16,550 of business income, a phantom half-SE-tax deduction and a $3,310 QBI deduction. It also deducted the repealed $1,796 DPAD and omitted the $13,899 overtime deduction, leaving $89,947 of taxable income instead of $67,812." -us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $18,235 capital loss instead of the $3,000 limit and omitted the $13,899 overtime deduction. It wrongly said the CTC/ODC were unavailable even though the MFJ phaseout starts at $400,000. It then reported $14,629.40 even though its own computed tax was $7,629.40." -us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It deducted the full $18,235 capital loss, took both itemized medical expenses and the standard deduction, and invented a $12,150 child tax credit. It then added that credit to tax instead of subtracting it. The correct credits are $2,200 CTC plus $1,000 ODC, subtracted from $7,641.46 of tax." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It reached the correct structure (AGI $114,422, $13,899 overtime deduction, $510 QBI, $3,200 of credits) and a pre-credit tax of about $7,630, which leaves about $4,430. It then added an unexplained ~$1,529 of 'adjustments' to report $5,959." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership/S-corp income, although the $14,000 is a component of it, which inflated gross income to $128,530. It also deducted the repealed $1,796 domestic production deduction and took a $3,310 QBI deduction instead of 20% × $2,550 = $510." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $13,899 overtime premium on top of wages that already include it, applied the full $18,235 capital loss instead of the $3,000 limit, and used a $14,600 single-filer standard deduction instead of the $32,200 MFJ amount. It also never took the OBBBA overtime deduction and finished with an unexplained credit reduction to $9,631." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It gave no calculation. Taxable income of $67,812 produces $7,641 of tax, and only $3,200 of nonrefundable CTC/ODC applies, so a $0 answer means it assumed deductions and credits that eliminate a liability that actually remains." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It computed SE tax on the $14,000 partnership income without netting the $16,800 farm loss (net SE earnings are negative, so there is no SE tax or half-SE-tax deduction) and omitted the $13,899 OBBBA overtime deduction. It also used a $2,000 CTC, capped its nonrefundable portion at $1,072 even though tax liability covers the full credit, and added an arbitrary $1,000." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction and took a half-SE-tax deduction even though the farm loss makes net SE earnings negative. It also added $1,978 of SE tax into income tax (this output excludes SE tax) and padded the result upward to $14,959, which is more than its own pre-credit tax." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It deducted the repealed $1,796 DPAD and a half-SE-tax deduction that does not exist here, took a $3,900 QBI deduction instead of $510, and omitted the $13,899 overtime deduction. It then reported $11,200, which is above its own ~$8,700 pre-credit tax, instead of subtracting the $3,200 of credits." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the OBBBA $13,899 FLSA overtime-premium deduction. It used the old $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 amount, and it subtracted a $1,560 SE health insurance deduction and a $989 half-SE-tax deduction that do not apply, because the premiums are employer coverage and the farm loss wipes out net SE earnings. It also used a $2,000 CTC instead of $2,200." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted the repealed $1,796 DPAD and a half-SE-tax amount, omitted the $13,899 overtime deduction, and used a $30,300 standard deduction. It then raised its own ~$9,300 tax to $12,500 by citing Additional Medicare Tax and NIIT, which do not belong in this output, and gave no credit for the $2,200 CTC and $1,000 ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its ~$128.5k AGI counts the $14,000 SE partnership income in addition to the $19,350 partnership income, which double-counts it, and it took a $3.3k QBI deduction instead of $510. It also omitted the $13,899 overtime deduction, which left taxable income at ~$92.9k instead of $67,812." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income in its $16,550 business net, deducted the repealed $1,796 DPAD, took a $3,310 QBI deduction, and omitted the $13,899 overtime deduction. It also treated the 18-year-old as a CTC-qualifying child ($4,000 for two children) instead of applying $2,200 for the 12-year-old plus $500 each for the two other dependents." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law, using personal exemptions, a $15,900 standard deduction, a 15% bracket and a $1,000 CTC, instead of the permanent OBBBA rules: $32,200 standard deduction, 10%/12% brackets, $2,200 CTC and $500 ODC. It also omitted the overtime deduction and wrongly subtracted $1,560 of health premiums from wages." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA rules (five personal exemptions and a $1,000 CTC phased out above $110,000) instead of the 2026 OBBBA $32,200 standard deduction, no exemptions, $2,200 CTC with a $400,000 phase-out threshold, and $500 ODC. It also omitted the $13,899 overtime deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It got AGI ($114,422) and QBI ($510) right but omitted the $13,899 OBBBA overtime-premium deduction and deducted the repealed $1,796 DPAD. It also used a $30,750 standard deduction instead of $32,200 and a $2,000 CTC instead of $2,200." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $19,648 employer insurance premium from wages, which the facts do not support, itemized medical expenses that include those premiums, and applied pre-TCJA personal exemptions and a $1,000 CTC. It also used the $1,796 DPAD as a QBI deduction and omitted the $13,899 overtime deduction and the $500 other-dependent credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $11,090 is more than the $7,641 pre-credit tax on the correct $67,812 of taxable income. That number fits an answer that omitted the $13,899 overtime deduction and never subtracted the $3,200 of CTC/ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly used pre-TCJA 2026 rules (personal exemptions, $16,200 standard deduction, $1,000 CTC), subtracted ESI premiums from wages, and deducted the repealed DPAD. It also added a $2,000 10% early-withdrawal penalty that is not part of this output and omitted the overtime deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied personal exemptions and a $1,000 CTC under the expired pre-TCJA regime, from an AGI understated at $94,756, instead of the OBBBA $32,200 standard deduction, $13,899 overtime deduction, $2,200 CTC and two $500 ODCs." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $11,940 is above the $7,641 pre-credit tax on the correct $67,812 of taxable income, which fits an answer that skipped the $13,899 overtime deduction and the $3,200 of nonrefundable credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Adding back its $3,200 of credits implies ~$9,415 of tax on ~$82,600 of taxable income, roughly the $67,812 correct figure plus the $13,899 overtime premium. It omitted the OBBBA overtime deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $4,324 implies a pre-credit tax of $7,524 on ~$66,830 of taxable income, about $980 below the correct $67,812. It overstated its below-AGI deductions by roughly that amount while otherwise applying the $32,200 standard deduction, overtime deduction and $3,200 of credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $4,057 implies ~$7,257 of pre-credit tax on ~$64,600 of taxable income, about $3,200 below the correct $67,812. That fits an inflated QBI deduction (20% of $16,550 instead of 20% of $2,550) or the repealed DPAD added on top of the correct deductions." +us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and credits wipe out the tax without computing anything. Taxable income is $67,812, giving $7,641 of tax, and only $3,200 of nonrefundable CTC/ODC applies, so $4,441 remains." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed liability is zero without computing it. After the $32,200 standard deduction, $13,899 overtime deduction and $510 QBI deduction, taxable income is still $67,812 ($7,641 of tax), and the $3,200 of credits does not eliminate it." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,219 implies ~$5,419 of pre-credit tax on ~$49,300 of taxable income, about $18,500 below the correct $67,812. That fits treating the $19,648 employer insurance premium as a reduction of taxable wages, which the facts do not support." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used the right components (2026 standard deduction, overtime deduction, $3,200 of credits), but its $44 excess equals about $360 of extra taxable income. One of its deduction amounts (QBI, IRA or the 401(k) exclusion) was understated by that much." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Its explanation subtracts only the qualifying-child credit, and $5,455 is about $1,000 above the correct $4,441. It dropped the two $500 credits for other dependents (the disabled 27-year-old and the 18-year-old)." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its $92,978 AGI equals the correct $114,422 less the $19,648 employer insurance premium and the $1,796 DPAD. It treated ESI premiums as a pre-tax wage reduction, which the facts do not support, and deducted the repealed domestic production deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $5,570 implies $8,770 of pre-credit tax on ~$77,200 of taxable income, about $9,400 above the correct $67,812. That fits double-counting the $14,000 SE partnership income on top of the $19,350 partnership income, partly offset by the DPAD and a $3,310 QBI deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It deducted the repealed $1,796 DPAD, which brought AGI to $112,626 instead of $114,422. It also claimed $1,500 of other-dependent credits instead of $1,000, since only Dependent 1 and Dependent 2 qualify for the $500 ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $13,500 is far above the $7,641 pre-credit tax on the correct $67,812 of taxable income, which fits an answer that omitted the overtime deduction, overstated income and applied no CTC/ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly applied TCJA-sunset rules ($17,000 standard deduction, $27,000 of personal exemptions, a 15% bracket and a $1,000 CTC phased out above $110,000) instead of the permanent OBBBA rules: $32,200 standard deduction, $13,899 overtime deduction, $2,200 CTC and two $500 ODCs." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its $114,523 figure adds the $14,000 SE partnership income on top of the $19,350 partnership/S-corp income, which double-counts it, and it took a $3,310 QBI deduction instead of $510. It also used a $30,700 standard deduction instead of $32,200." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income in AGI and took a $3,310 QBI deduction instead of $510. It also added a $2,000 early-distribution penalty that is not part of this output." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 of ESI and health premiums from income, double-counted the $14,000 SE partnership income, took a half-SE-tax deduction, and used the $1,796 DPAD as QBI. It also omitted the $13,899 overtime deduction and claimed only a $2,000 CTC, with no $500 credits for the two other dependents." +us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added a $2,000 early-withdrawal penalty on the 401(k) distribution, which is not part of this output. It also used an understated $97,723 AGI and deducted the repealed $1,796 DPAD instead of the $510 QBI deduction and $13,899 overtime deduction applied to the $114,421 AGI." +us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It double-counted the $14,000 SE partnership income, deducted Dependent 1's own 401(k) and IRA contributions on the parents' return, and took a $6,670 QBI deduction on $33,350. It also omitted the $13,899 overtime deduction and used a $2,000 CTC with only one $500 ODC, leaving out the 18-year-old's ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It never excluded the $2,315 traditional 401(k) deferral from wages, so AGI came out at $116,737 instead of $114,421. It also deducted the repealed $1,796 DPAD instead of the $510 QBI deduction, which left taxable income $1,030 too high." +us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It gave $0 with no reasoning. The correct $67,812 of taxable income produces $7,641 of tax, and only $3,200 of nonrefundable CTC/ODC offsets it, so a zero answer means it assumed a liability-eliminating deduction or credit that does not exist." +us,scenario_064,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It applied the correct structure (standard, overtime and QBI deductions and $3,200 of credits), but its $23 excess equals about $190 of extra taxable income. One of its deduction inputs was slightly understated compared with the $67,812 taxable-income result." +us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income in addition to the $19,350 partnership income and took a half-SE-tax deduction. It also deducted the repealed $1,796 DPAD, took a $3,310 QBI deduction instead of $510, and omitted the $13,899 overtime deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $18,235 capital loss instead of the $3,000 limit and omitted the overtime, QBI and IRA deductions. It denied the $3,200 of CTC/ODC (the $400,000 phase-out threshold is far away), then reported $14,629.40 even though its own tax figure was $7,629.40." +us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It deducted the full capital loss and $11,827 of retirement contributions that include employee Roth and dependents' amounts, and it claimed both itemized medical expenses and the standard deduction. It then added a fictitious $12,150 child tax credit to the tax instead of subtracting the correct $3,200 of CTC/ODC." us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It equated the refundable credit with the full $2,000 CTC for Child 1, skipping the ACTC computation entirely — the CTC first offsets federal income tax liability and only the unused remainder becomes refundable, capped at $1,700 per child and at 15% of earned income over $2,500. With $97,295 of wages plus $20,000 of taxable 401(k) distributions and $19,350 of partnership income, MFJ tax before refundable credits far exceeds $2,000, so the entire CTC is consumed nonrefundably and nothing spills into the refundable column. Its own AGI estimate of ~$120,610 also double-counted, adding partnership income while ignoring that the capital loss deduction is capped at $3,000." us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning reached the correct conclusion — 'with significant federal tax owed, CTC is nonrefundable' — and it then submitted the $1,700 statutory ACTC ceiling anyway, converting a per-child maximum into an entitlement. The ACTC equals only the CTC remaining after the credit is applied against federal income tax liability; here liability on ~$98,810 of income absorbs the full $2,000 for the one under-17 child, so the refundable amount is $0." us,scenario_064,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It counted Dependent 2 (age 18) as a CTC qualifying child, but the credit requires age under 17 at year end, so that dependent yields only the $500 nonrefundable other-dependent credit rather than a $1,700 refundable share. It then paid out the per-child ACTC ceiling without applying the rule that only the CTC unused against tax liability is refundable; MFJ tax before refundable credits on ~$98,810 of income exceeds $2,000, so even the single qualifying child's credit is fully nonrefundable and the correct total is $0. Checking the $400,000 phaseout threshold was irrelevant — the binding constraint is the liability offset, not the income phaseout." @@ -5405,52 +5464,53 @@ us,scenario_064,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model us,scenario_064,ssi,inkling,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled, applied the one-third reduction for living in another person's household ($644.67 × 12 ≈ $7,736) and added an estimated Wisconsin supplement to reach $7,952. Dependent 1 fails the engine's SSI aged/blind/disabled test (is_ssi_aged_blind_disabled=False), so no federal benefit or state supplement is paid and SSI is $0." us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SSI value and no explanation, so there is no answer to score. The correct derivation is $0: no member meets SSI's aged/blind/disabled requirement under the engine's inputs." us,scenario_064,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model correctly excluded Head and Spouse as neither 65+ nor disabled, but wrongly declared Dependent 1 'categorically SSI-eligible' because of the disability flag and paid $994 × 12 = $11,928. The engine's SSI aged/blind/disabled test is also False for Dependent 1, so no member is eligible and SSI is $0." -us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It added back $2,500 on the belief that Wisconsin caps net capital losses at $500, but Wisconsin uses the same $3,000 joint limit. It also deducted half of an SE tax that doesn't exist, because the $16,800 farm loss wipes out the $14,000 SE partnership income. After taxing at the old schedule where 5.3% starts at $38,190 (giving $4,964), it cut the result to $2,699 using an itemized-deduction credit and unexplained adjustments, even though no nonrefundable credits apply." -us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its $126,626 federal AGI counts the $14,000 SE partnership income a second time, even though it is already inside the $19,350 partnership/S-corp income, and wrongly subtracts the $1,796 domestic production deduction. It then added $2,500 more under a nonexistent $500 Wisconsin capital-loss cap. That inflated taxable income to about $117,700 instead of $101,968.14." -us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It started from 'federal taxable income' that subtracts the full $18,235 capital loss instead of the $3,000 limit, and it cited a made-up 5.3%–7.65% Wisconsin rate range. It never applied the $8,953.27 sliding standard deduction and five $700 exemptions to the $114,421.41 Wisconsin AGI, so $4,512 is an unsupported guess." -us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It explicitly skipped the Wisconsin computation and reported $0. With $114,421.41 of Wisconsin AGI, the $8,953.27 sliding standard deduction and $3,500 of exemptions, this joint return has $101,968.14 taxable and owes $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It left out the $2,315 401(k) wage exclusion and the $108 IRA deduction (AGI $116,845 instead of $114,421.41), and it counted four $700 exemptions instead of five. It used the outdated schedule where 5.3% starts at $36,840, not the expanded 4.4% bracket running to roughly $69,000. It then subtracted about $875 of school property tax and itemized deduction credits, which don't apply: no property tax or rent is listed and the return uses the standard deduction." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied the pre-2025 joint brackets, where 5.3% starts at $38,190, instead of the 2026 schedule where 4.4% runs to roughly $69,000, and got $4,784. It then raised the answer to $5,074 while claiming exemptions and the standard deduction reduce tax, so the final number contradicts its own computation." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It added back $2,500 on the assumption of a $500 Wisconsin capital-loss limit. Wisconsin allows $3,000 on a joint return, so Wisconsin AGI equals the federal $114,421.41. It then taxed the inflated $105,422 using the old schedule where 5.3% starts near $38,000, not the expanded 4.4% bracket to roughly $69,000." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It deducted a $990 half-SE-tax that doesn't exist, because the farm loss offsets the SE partnership income. It also disallowed the $108 IRA deduction, added back $2,500 under a nonexistent $500 capital-loss cap, and used a made-up standard-deduction phase-out ($22,010 phased out linearly by $134,000), which gave $3,957 instead of $8,953.27. It then taxed at the old brackets and subtracted $520 of invented credits." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Its $126,626 AGI counts the $14,000 SE partnership income twice, even though it is already part of the $19,350 partnership/S-corp income, and subtracts the $1,796 domestic production deduction, which is not an AGI adjustment. It also used about $14,000 of standard deduction and only four exemptions, and its $4,200 doesn't follow from its own $109,826 taxable income at the rates it named." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It put Wisconsin taxable income at about $125,000 instead of $101,968.14, which means it overstated income by about $23,000. It also allowed only three $700 dependent exemptions, leaving out the head's and spouse's exemptions (five in total), and taxed the excess at 5.3%." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a made-up two-rate schedule (4% on the first $23,000 and 5.3% on the rest) instead of Wisconsin's 3.5%/4.4%/5.3% brackets, where 4.4% runs to roughly $69,000 for joint filers. It also subtracted only the standard deduction and skipped the $3,500 of personal exemptions." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It took a flat $25,000 standard deduction, ignoring Wisconsin's sliding-scale phase-down, which cuts the joint deduction to $8,953.27 at $114,421.41 of income. It also subtracted the $1,796 domestic production deduction from Wisconsin AGI. Its taxable income of $82,566 is about $19,400 too low." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its Wisconsin taxable income of $119,774.69 is $17,806 above the correct $101,968.14, a gap about the size of the $16,800 farm loss, which it did not subtract from income. Taxing that excess at 5.3% produced the $1,230 overstatement." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It treated the five $700 personal exemptions as a $3,500 credit against tax instead of a $3,500 deduction from income, which removed $3,500 directly from the liability. It also used the outdated 3.54%/4.65% rates and an overstated $13,197.25 standard deduction instead of $8,953.27." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It overstated the sliding-scale standard deduction ($13,506 instead of $8,953.27). It used the pre-2025 brackets, where 5.3% starts at $38,190, instead of the 2026 schedule with 4.4% to roughly $69,000. It then subtracted a $150 'dependent credit' that Wisconsin does not have." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is $114,421.41 of Wisconsin AGI, minus the $8,953.27 sliding standard deduction and $3,500 of exemptions, giving $101,968.14 taxable and $4,599.62 of tax. Its $4,652.12 is $52.50 higher, which matches about $990 of extra taxable income at the 5.3% marginal rate, consistent with phasing the standard deduction down too far." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from a $92,978 AGI, $21,443 below the correct $114,421.41. That left taxable income at $78,969 instead of $101,968.14, so the tax came out about $930 too low." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $82,256 of Wisconsin taxable income is about $19,700 below the correct $101,968.14, so it understated income or overstated deductions well beyond the $8,953.27 sliding standard deduction and $3,500 of exemptions. Its $4,356 also doesn't follow from that base under the 3.5%/4.4%/5.3% schedule." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $6,950 is $2,350 above the correct $4,599.62, which requires about $44,000 more taxable income than the correct $101,968.14 at the 5.3% marginal rate. It therefore failed to subtract the farm loss, the $3,000 capital loss, the $8,953.27 sliding standard deduction and the $3,500 of exemptions from Wisconsin income." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,125 is $475 below the correct $4,599.62, which equals about $9,000 of extra deductions at the 5.3% marginal rate. That is consistent with an under-phased standard deduction of about $18,000 instead of the $8,953.27 the sliding scale allows at $114,421.41 of income." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,212 is $1,388 below the correct $4,599.62, which equals about $26,000 of excess deductions at 5.3%. Only the $8,953.27 sliding standard deduction and $3,500 of exemptions apply, with no nonrefundable credits." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,490 is $2,110 below the correct $4,599.62, which equals about $40,000 of excess deductions or invented credits. Only the $8,953.27 sliding standard deduction and $3,500 of exemptions reduce the $114,421.41 Wisconsin AGI, and no nonrefundable credits apply." -us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions and retirement adjustments reduce Wisconsin tax to zero. In fact the sliding standard deduction ($8,953.27) and five exemptions ($3,500) total only $12,453.27 against $114,421.41 of Wisconsin AGI, leaving $101,968.14 taxable." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits fully offset Wisconsin tax. In fact only the $8,953.27 sliding standard deduction and $3,500 of exemptions apply against $114,421.41 of Wisconsin AGI, and no nonrefundable credits apply, so $4,599.62 is owed." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It left out the $3,000 capital-loss deduction (using $117,222 of income instead of $114,421.41). It overstated the standard deduction at $15,166 instead of $8,953.27 and allowed only two $700 exemptions instead of five. It also subtracted a $472 itemized-deduction credit, which is zero here because qualifying medical costs don't clear the 7.5%-of-AGI floor and no mortgage interest is listed." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,800 is about $1,800 below the correct $4,599.62, which equals about $34,000 of excess deductions or invented credits. Only the $8,953.27 sliding standard deduction and $3,500 of exemptions apply to the $114,421.41 Wisconsin AGI, with no nonrefundable credits." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,872 matches the pre-2025 joint brackets, where 5.3% starts near $38,190; those brackets give about $4,889 on $101,968.14. Under the 2026 schedule the 4.4% rate runs to roughly $69,000, which brings the tax to $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated Wisconsin's personal exemptions as 'exemption credits' instead of $700 deductions from income for each of the five people. Its $4,071 is $528.62 below the correct $4,599.62, which equals about $10,000 of excess deductions at the 5.3% marginal rate." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It added back $2,500 under an outdated $500 Wisconsin capital-loss cap, but Wisconsin uses the $3,000 joint limit. It also took a $2,340 medical-care-insurance subtraction, which does not apply because the head and dependent have employer-sponsored coverage where the employer pays part of the premium. Its 'estimated indexed' brackets then put the tax about $1,345 too low on a taxable base near $102,000." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $5,654 is $1,054 above the correct $4,599.62, which equals about $20,000 more taxable income than $101,968.14 at 5.3%. That is consistent with leaving out the $8,953.27 sliding standard deduction and $3,500 of exemptions and using the pre-2025 brackets." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It started from the correct $114,421.41 AGI, but its $4,200 is about $400 too low, which equals roughly $7,500 of extra deductions at 5.3%. It used a standard deduction well above the $8,953.27 that the sliding scale allows at this income." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It subtracted a $4,136 health-premium deduction to reach $110,286, but Wisconsin's medical care insurance subtraction does not apply to employer-sponsored coverage where the employer pays part of the premium, so Wisconsin AGI stays at $114,421.41. The lower base also pushed the sliding standard deduction up to $10,207.94 instead of $8,953.27." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,500 is $99.62 below the correct $4,599.62, which equals about $1,880 of excess deductions at 5.3%. That is consistent with a standard deduction above the $8,953.27 sliding-scale amount at $114,421.41 of income." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It left out the five $700 personal exemptions ($3,500) entirely, computing taxable income as $114,422 minus only the standard deduction. It also understated the sliding standard deduction at about $6,500 instead of $8,953.27, leaving taxable income about $5,900 too high." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its standard deduction, exemptions and taxable income ($101,947) were essentially right, but it used made-up equal-width brackets of $29,350 each. That made the 3.5% bracket too wide and ended the 4.4% bracket at $58,700 instead of the 2026 joint threshold of roughly $69,000, producing $4,610 instead of $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its taxable income of about $102,274 is close, but the $4,876 comes from the pre-2025 brackets, where 5.3% starts near $38,000–$39,000. The 2026 joint schedule applies 4.4% up to roughly $69,000, which brings the tax down to $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $106,225 AGI (about $8,200 too low), an unreduced $13,500 standard deduction, and no personal exemptions. It also applied made-up 4.75%/5.75% rates on 2024 thresholds instead of Wisconsin's 3.5%/4.4%/5.3% schedule." -us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It started from a $97,723 federal AGI and added back $13,899 for the federal overtime deduction. That deduction is taken below AGI, so federal AGI already includes all wages and equals $114,421.41. Getting from $111,622 down to about $90,000 of taxable income took about $21,600 of deductions, versus the correct $12,453.27 ($8,953.27 sliding standard deduction plus $3,500 of exemptions)." -us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $4,599.62." -us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied a 6.27% third-bracket rate that doesn't exist; Wisconsin's rate is 5.3%. Its $116,737 AGI also leaves out the $2,315 traditional 401(k) wage exclusion, and its $5,481 sliding standard deduction is too small compared with $8,953.27." -us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It reported $0 with no derivation. With $114,421.41 of Wisconsin AGI, the $8,953.27 sliding standard deduction and $3,500 of exemptions, taxable income is $101,968.14 and the tax is $4,599.62, with no nonrefundable credits to offset it." -us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It used roughly the federal joint standard deduction ($31,300) instead of Wisconsin's sliding-scale deduction, which phases down to $8,953.27, and it left out the $3,500 of exemptions. Its $127,253 AGI is about $12,800 too high, and its $29,430/$58,860 bracket thresholds are wrong. These errors partly offset each other, giving $4,291." -us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It removed $13,899 of FLSA overtime from wages when computing AGI, but the federal overtime deduction is taken below AGI and Wisconsin does not adopt it, so those wages stay in Wisconsin AGI. It also used an unreduced $13,500 standard deduction, left out the $3,500 of exemptions, used a 4.65% rate, and reported $4,221.67 even though its own arithmetic gave $3,767." -us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted $11,827 of 'retirement contributions' after AGI, but the $2,315 traditional 401(k) deferral is already excluded from wages and Roth contributions are never deductible. It also deducted medical expenses, which Wisconsin does not allow as a deduction from income. Its $101,610 AGI starting point is already about $12,800 too low." +us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It wrongly subtracted a $989 self-employment tax deduction, but the $16,800 farm loss offsets the $14,000 self-employment partnership income, so there is no self-employment tax. It also added back $2,500 under a $500 Wisconsin capital-loss cap and skipped the $2,315 401(k) exclusion. It then cut its own $4,964 gross tax to $2,699 with an invented itemized-deduction credit and unexplained 'precision' adjustments. The household has no nonrefundable credits, so the tax should be $4,599.62." +us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $126,626, which counts the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp income that already includes it. It also added back $2,500 under a $500 capital-loss cap. That gave taxable income of about $117,700 instead of $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built taxable income by subtracting the full $18,235 capital loss instead of the $3,000 limit. It then cited a 5.3%–7.65% rate schedule, leaving out the 3.5% and 4.4% brackets. The correct calculation is federal AGI of $114,421.41, minus the $8,953.27 sliding-scale deduction and $3,500 of exemptions, giving $101,968.14 taxable." +us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It openly skipped the Wisconsin calculation and entered 0. The joint return actually owes $4,599.62 on $101,968.14 of taxable income." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It counted only four $700 exemptions instead of five and left out the $2,315 401(k) and $108 IRA reductions to AGI. It then subtracted about $875 for school property tax and itemized deduction credits. The household lists no property tax or rent and has no itemized deductions above the standard deduction, so no nonrefundable credits apply." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the old 4.4% bracket ceiling of $38,190, but the 2026 joint bracket runs to roughly $69,000. That overtaxed the income between those levels at 5.3%. It then raised its own $4,784 figure to $5,074 with no supporting calculation." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It added back $2,500 by applying a $500 Wisconsin capital-loss cap. Under the 2026 rules the $3,000 federal deduction carries over, so Wisconsin AGI equals federal AGI of $114,421.41. The addback, together with a slightly smaller standard deduction of about $8,000, raised taxable income to $105,422 instead of $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted a $990 self-employment tax deduction that doesn't exist here, because the farm loss eliminates net self-employment earnings. It disallowed the $108 IRA deduction and added back $2,500 under a $500 capital-loss cap. It phased the standard deduction down to $3,957 instead of $8,953.27, used the old narrow brackets, and then subtracted $520 of made-up credits." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp income and subtracted the repealed $1,796 domestic production deduction. It used a $14,000 standard deduction and only four exemptions. Its $4,200 does not even follow from its own $109,826 taxable income." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It put taxable income at about $125,000 instead of $101,968.14, which overstates income from AGI of $114,421.41. It also allowed only three dependent exemptions, dropping the $700 exemptions for the head and spouse." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied a made-up schedule: 4% on the first $23,000 and 5.3% on everything above. The 2026 joint schedule is 3.5%, then 4.4% up to roughly $69,000, then 5.3%. Its schedule taxes about $46,000 at 5.3% that should be taxed at 4.4%." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the repealed $1,796 domestic production deduction and used a flat $25,000 standard deduction. Wisconsin's sliding-scale deduction phases down to $8,953.27 at this income, so taxable income fell to $82,566 instead of $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its taxable income of $119,774.69 is $17,806.55 above the correct $101,968.14. That gap roughly matches the $16,800 farm loss, which it failed to net. Its tax on that base also reflects the old narrow 4.4% bracket rather than the 2026 bracket running to about $69,000." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It treated the five $700 Wisconsin exemptions as a $3,500 dollar-for-dollar tax credit, but they are deductions from income. It also used outdated 3.54%/4.65% rates and an overstated $13,197 standard deduction instead of $8,953.27." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It overstated the sliding-scale standard deduction as $13,506 instead of $8,953.27 and used the old narrow 4.4% bracket. It then subtracted a $150 'dependent credit' that does not exist. Wisconsin allows no nonrefundable credits for this household." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no calculation, and its $4,652.12 is $52.50 above the correct tax. That gap equals about $990 of extra taxable income at the 5.3% marginal rate, the size of the self-employment tax deduction that isn't owed here. The correct tax applies the 2026 schedule to $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It understated AGI as $92,978 instead of $114,421.41, a $21,443 shortfall, so taxable income was about $78,969 instead of $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It put net taxable income at $82,256 instead of $101,968.14, nearly $20,000 too low, by over-deducting from AGI of $114,421.41. The sliding-scale deduction is only $8,953.27, and there are $3,500 of exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no calculation, and its $6,950 implies taxable income of about $146,000. That is consistent with not netting the $16,800 farm loss and $3,000 capital loss and skipping the sliding-scale deduction and exemptions. The correct taxable income is $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no calculation, and its $4,125 is $474.62 too low. That matches about $8,955 of extra deductions at the 5.3% marginal rate, which means it used roughly double the phased-down $8,953.27 standard deduction." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no calculation, and its $3,212 is $1,387.62 too low. That implies about $26,000 of extra deductions or invented nonrefundable credits. At this income the standard deduction is only $8,953.27, and the household has no nonrefundable credits." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no calculation, and its $2,490 is $2,109.62 too low. That is consistent with using the full, unphased standard deduction instead of the sliding-scale $8,953.27, or subtracting credits the household doesn't have." +us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no usable value for state_income_tax_before_refundable_credits, so there was nothing to score against $4,599.62." +us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It submitted no usable value for state_income_tax_before_refundable_credits, so there was nothing to score against $4,599.62." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions and retirement adjustments wipe out the tax. In fact, after the $8,953.27 sliding-scale deduction and $3,500 of exemptions, $101,968.14 of income remains taxable, and no credits offset the $4,599.62." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits cancel the tax. Federal AGI of $114,421.41 leaves $101,968.14 taxable after the sliding-scale deduction and exemptions, and Wisconsin offers this household no nonrefundable credits." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It left the $3,000 capital loss out of its $117,222 income and used a $15,166 standard deduction instead of $8,953.27. It allowed only two $700 exemptions instead of five. It then subtracted a $472 itemized-deduction credit, but medical costs are below the floor and no mortgage interest is listed, so no credit applies." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"It gave no calculation, and its $2,800 is $1,799.62 too low. That implies heavy over-deduction or invented nonrefundable credits. The household has $101,968.14 of taxable income and no credits." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,872 is $272.38 above the correct tax. That matches using the old 4.4% bracket ceiling of about $40,000 instead of the 2026 joint ceiling of roughly $69,000, which taxes about $29,000 at 5.3% instead of 4.4%." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied 'exemption credits' against the tax. Wisconsin's $700 personal exemptions are deductions from income, not credits, and no nonrefundable credits apply. That cut the tax $528.62 below $4,599.62." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It added back $2,500 under a $500 capital-loss cap and subtracted $2,340 of health insurance premiums. Wisconsin AGI equals federal AGI here, with no medical-insurance subtraction for employer-covered workers. Its 'estimated indexed' brackets were also too low, producing $3,254.36 when taxable income near $102,000 gives about $4,600." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no calculation, and its $5,654 is $1,054.38 too high. That equals about $19,900 of extra taxable income at 5.3%, consistent with double counting the $14,000 self-employment partnership income or not netting the farm loss. The correct taxable income is $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It started from the correct AGI of $114,421, but its $4,200 is $399.62 too low. That implies about $7,500 more in deductions than the $8,953.27 sliding-scale deduction plus five $700 exemptions allow." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted a $4,135 health-premium deduction to reach $110,286. No such Wisconsin subtraction applies to these employer-covered premiums. That lowered taxable income and inflated the standard deduction to $10,207.94, and its rate arithmetic also used a narrower 4.4% bracket than the 2026 schedule." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no calculation, and its round $4,500 is $99.62 short. It never applied the 2026 joint schedule precisely to the $101,968.14 left after the $8,953.27 deduction and $3,500 of exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It left out the five $700 personal exemptions entirely and used a sliding-scale deduction of about $6,500 instead of $8,953.27. That put taxable income at $107,900 instead of $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its taxable income of $101,947 is essentially correct, but it used bracket breaks of $29,350 and $58,700. The 2026 joint schedule breaks at about $20,000 and $69,000, so it overstated the tax by $10.38." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its taxable income of about $102,274 is close, but its $4,876 implies the old 4.4% bracket ending near $40,000. The 2026 joint bracket runs to roughly $69,000, so it taxed about $29,000 too much at 5.3%." +us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 2024 brackets with 4.75% and 5.75% rates, which are not in the 2026 schedule of 3.5%/4.4%/5.3%. It also understated AGI at $106,225, used a $13,500 standard deduction instead of $8,953.27, and left out the $3,500 of exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It took the federal overtime deduction out of AGI and then added it back. That deduction never reduces AGI, so its starting AGI of $97,723 is wrong; the correct federal AGI is $114,421.41. It then over-deducted to about $90,000 of taxable income instead of $101,968.14." +us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no usable value for state_income_tax_before_refundable_credits, so there was nothing to score against $4,599.62." +us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied an obsolete 6.27% rate to the third bracket, but the 2026 rate is 5.3%. It also left the $2,315 401(k) deferral out of AGI and understated the sliding-scale deduction at $5,481 instead of $8,953.27." +us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It entered 0 with no calculation. The joint return has $101,968.14 of Wisconsin taxable income and owes $4,599.62 with no offsetting nonrefundable credits." +us,scenario_064,state_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It gave only a vague description, and its $4,448 is $151.62 short. It never applied the 2026 joint schedule to the exact $101,968.14 left after the $8,953.27 sliding-scale deduction and $3,500 of exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a $31,300 federal-style standard deduction instead of Wisconsin's sliding-scale $8,953.27 and left out the five $700 exemptions. It overstated AGI at $127,253 instead of $114,421.41 and used made-up bracket breaks of $29,430 and $58,860." +us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It excluded the $13,899 overtime premium from federal AGI, but that deduction does not reduce AGI and Wisconsin does not adopt it. It used a $13,500 standard deduction, no exemptions, and an obsolete 4.65% rate. It then reported $4,221.67, which does not match its own $3,767." +us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted $11,827 of retirement contributions, including non-deductible Roth amounts and a 401(k) deferral already excluded from wages. It also deducted medical expenses, which Wisconsin does not allow as a deduction. Both pushed taxable income far below $101,968.14." us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model enumerated every Wisconsin refundable credit — Homestead (income too high), the WI EITC (federal EITC is $0, and the WI credit is a straight percentage of it), Farmland Preservation, and the Veterans and Surviving Spouses Property Tax Credit — and stated that each one does not apply, then submitted $154 rather than the $0 its own enumeration produces. No credit it named yields $154: the Homestead Credit is additionally barred because no rent or property tax is listed, and the Married Couple and Itemized Deduction credits it correctly identified as nonrefundable are excluded from this output entirely." us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the state_refundable_credits key from its outputs object, so it submitted no value for a variable the prompt required exactly once. The correct derivation sums Wisconsin's refundable credits — WI EITC at $0 because the federal EITC is $0 on $97,295 of wages, Homestead at $0 under its household-income ceiling and with no rent or property tax listed, Farmland Preservation at $0 absent a preservation agreement — for a total of $0." us,scenario_064,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no state_refundable_credits entry, leaving the required key unanswered rather than making a substantive Wisconsin credit error. Working the rule set through yields $0: the WI EITC is a percentage of a federal EITC that is $0 at this income, the Homestead Credit fails both the income ceiling and the requirement that rent or property tax be paid, and the Farmland Preservation Credit requires an unlisted farmland preservation agreement." @@ -5563,105 +5623,108 @@ us,scenario_067,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household us,scenario_067,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It chose the right program, Indiana HIP for adults up to 138% FPL, but tested dependent1 against the household's roughly $87K MAGI. PolicyEngine puts dependent1's own MAGI income level at 0.00×FPL, so dependent1 qualifies." us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It wrongly treated Indiana as limited on expansion-type pathways and tested dependent1 against 3-person household MAGI of about $87,302. Indiana has adopted the ACA expansion, and PolicyEngine puts dependent1 in the ADULT category at 0.00×FPL, so dependent1 is eligible." us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed dependent1's $10,800 in disability benefits exceed Indiana's adult Medicaid limit. The benefits are not MAGI income, so PolicyEngine puts dependent1 at 0.00×FPL, and $10,800 alone is still under 138% FPL for a single adult." -us,scenario_067,dependent1_medicare_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated being disabled plus receiving disability benefits as enough for Medicare. The under-65 pathway actually requires 24 months of SSDI entitlement (or ESRD/ALS). The listed disability benefits are generic, not SSDI, and months of SSDI receipt are 0, so the 23-year-old does not qualify." -us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model claimed a disability-based Medicare rule is triggered by disabled status plus disability benefits. Medicare eligibility under 65 requires 24 months of Social Security disability entitlement. The generic disability-benefits income here is not SSDI, and SSDI months are 0, so Dependent 1 is not eligible." -us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly named the 24-month SSDI waiting period, then decided the $10,800 was SSDI and assumed the waiting period was met. Both steps go against the instructions not to infer unlisted benefit receipt and to treat unlisted numeric inputs (months of SSDI receipt) as 0. With no SSDI and 0 qualifying months, Dependent 1 is not Medicare-eligible." -us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model relabeled the generic disability benefits as SSDI and treated the 24-month waiting period as satisfied. Neither fact is listed, and SSDI months default to 0. Without 24 months of SSDI entitlement or ESRD, a 23-year-old does not qualify for Medicare." -us,scenario_067,dependent1_medicare_eligible,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model went straight from ""disabled"" to ""Medicare eligible."" It skipped the requirement that people under 65 need 24 months of SSDI entitlement (or ESRD/ALS). Dependent 1 has 0 SSDI months and only generic disability income, so the person is not eligible." -us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated annual disability benefits as support for Medicare eligibility under 65. The disability pathway requires 24 months of Social Security disability entitlement specifically, not generic disability income. SSDI months are 0 here, so the answer is not eligible." -us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model assumed disabled status plus disability benefits triggers disability-based Medicare. That pathway requires 24 months of SSDI entitlement. The listed benefits are a generic disability-income input, not SSDI, and SSDI months are 0, so Dependent 1 is not eligible." -us,scenario_067,dependent1_medicare_eligible,gpt-6-astra,llm_error,categorical_eligibility,False,"The model applied a disability-based Medicare pathway based only on reported disability and disability income. That pathway requires 24 months of SSDI receipt, which is 0 for this household, and the generic disability benefits do not count. The 23-year-old dependent is not Medicare-eligible." -us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model claimed that disability plus disability benefits qualifies a person under 65 for Medicare. Medicare actually requires 24 months of SSDI entitlement or ESRD/ALS. Dependent 1's disability benefits are not SSDI and SSDI months are 0, so the person is not eligible." -us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied ""disability rules"" without the actual condition, which is 24 months of Social Security disability entitlement before age 65. The generic disability benefits and the disabled flag do not meet that test, and SSDI months are 0. Dependent 1 is not Medicare-eligible." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model took a $1,845 OBBBA auto-loan-interest deduction that isn't available here and used a $32,600 standard deduction instead of the $32,200 2026 MFJ amount. Its own work came to $5,347, but it submitted $3,391, a number that no step of its reasoning supports." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Everything matched the reference except one step: it subtracted the $1,845 household auto loan interest as an OBBBA qualified vehicle loan interest deduction. That deduction requires facts the prompt never gives (a new, U.S.-assembled vehicle financed after 2024), so taxable income stays at $55,102. The extra deduction is exactly why it came in $221.40 low (12% × $1,845)." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 MFJ standard deduction instead of OBBBA's $32,200 for 2026 and estimated about $7,080 of pre-credit tax instead of $6,116.24. It then subtracted $1,266 of credits it never identified, when the only nonrefundable credit is the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Its rough $6,900 estimate overstates the tax on $55,102 of taxable income, which is $6,116.24 using the $32,200 standard deduction and the 10%/12% brackets with a $24,800 breakpoint. It also explicitly assumed no nonrefundable credits, so it never subtracted the $500 credit for the disabled adult dependent." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It reached about $5,620 after the $500 other-dependent credit, then 'adjusted' to $6,280 using a $32,300 standard deduction and 2025 brackets. Those inputs actually give about $5,623, so $6,280 isn't supported by anything in its reasoning. The correct inputs are the $32,200 standard deduction and a $24,800 10% bracket, which give $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It made up about $6,500 of mortgage interest from the $100,000 loan balance, even though unlisted interest counts as $0, and then itemized instead of taking the larger $32,200 standard deduction. Its own computation reached $6,722, but it submitted $5,594, which none of its steps support." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It got the correct path: $55,102 of taxable income, $6,116 of tax, and $5,616 after the $500 other-dependent credit. It then dropped that result for an unexplained 'bracket estimate' of $6,104, which amounts to not taking the $500 credit." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"On top of the $32,200 standard deduction, it subtracted the $1,845 auto loan interest as a qualified vehicle loan interest deduction. That deduction isn't available because the prompt doesn't establish the new, U.S.-assembled, post-2024-loan conditions. As a result, taxable income was $221.40 of tax too low (12% × $1,845)." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflation-adjusted pre-OBBBA 2025 figures instead of using OBBBA's 2026 parameters. That gave a $30,000 MFJ standard deduction instead of $32,200, which put $2,200 too much in the 12% bracket, and a $24,500 10% bracket instead of $24,800. Those changes produce its $5,886 result." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $6,068 of tax using a $32,600 standard deduction (the correct figure is $32,200) and acknowledged the $500 other-dependent credit. It then submitted $10,500, citing 'self-employment-like adjustments' and 'benefit phase-outs' that have no place in a wage-and-pension return." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 of nontaxable disability benefits to the couple's AGI and used a $30,600 standard deduction instead of $32,200. It also said no nonrefundable credit applies because the dependent is over 17. That misses the $500 credit for other dependents: as a permanently disabled qualifying child, the dependent is eligible regardless of age." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the expired pre-TCJA regime ($16,500 standard deduction, three $5,250 personal exemptions, 15% bracket). OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no exemptions, and a 12% second bracket. It also left out the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset: it itemized medical expenses plus state tax, took three $5,300 personal exemptions, and applied a 15% bracket. Under OBBBA, 2026 uses the $32,200 standard deduction (larger than its $19,108 of itemized deductions), no exemptions, and 10%/12% brackets. It also never subtracted the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted an auto-loan-interest deduction that doesn't apply, because the qualified-vehicle conditions aren't established, which pushed taxable income below the correct $55,102. Without that deduction the tax is $6,116.24, and after the $500 other-dependent credit it is $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the pre-TCJA regime (personal exemptions and a 15% bracket) instead of OBBBA's 2026 $32,200 standard deduction with 10%/12% brackets. It also treated the dependent's disability benefits as Social Security and added $9,180 of 'taxable' benefits to AGI. Finally, it didn't apply the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $31,200 standard deduction instead of $32,200, and its stated $60,400 of taxable income doesn't even match $87,302 minus that deduction. It also said there were no significant nonrefundable credits, which leaves out the $500 credit for the disabled adult dependent." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It wrongly treated 2026 as a TCJA-expiration year: it itemized $19,064, subtracted $16,050 of personal exemptions, and taxed income at 15%. OBBBA keeps the $32,200 MFJ standard deduction, no exemptions, and 10%/12% brackets, and it also allows the $500 other-dependent credit, which the model didn't apply." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA rules (personal exemptions and a 15% bracket) instead of OBBBA's 2026 $32,200 standard deduction and 10%/12% brackets. It also invented $4,500 of mortgage interest, although only the loan balance was given and unlisted interest counts as $0." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The correct calculation is $87,302 − $32,200 = $55,102 of taxable income, $6,116.24 of tax at 10%/12% with the $24,800 breakpoint, and $5,616.24 after the $500 other-dependent credit. Its $7,534 overstates taxable income and does not subtract that credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of OBBBA's $32,200 for 2026 MFJ, and it ended the 10% bracket at $23,850 (the 2025 figure) instead of $24,800. Together these overstated tax before the correctly applied $500 credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The rest of its structure was right (MFJ AGI and the $500 other-dependent credit). Its $5,912.24 matches stale parameters, roughly a $30,000 standard deduction and an earlier-year 10% bracket. With 2026's $32,200 standard deduction and $24,800 10% bracket, the result is $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used pre-OBBBA 2025 figures: a $30,000 standard deduction and a $23,850 10% bracket. The 2026 MFJ figures are $32,200 and $24,800, so its taxable income was $2,200 too high and too little income fell in the 10% bracket." -us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction and a $23,600 '2025 proxy' for the end of the 10% bracket. OBBBA's 2026 MFJ figures are a $32,200 standard deduction and a $24,800 10% bracket, so its taxable income and tax were both too high." -us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied the $500 credit for other dependents by applying the qualifying-relative gross-income test. A permanently and totally disabled child of any age is a qualifying child, so that test doesn't apply, and the dependent qualifies for the credit. It also ended the 10% bracket at $25,200 instead of $24,800." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $4,567 understates the liability. With the $32,200 standard deduction, taxable income is $55,102 and tax is $6,116.24 at 10%/12%. The only nonrefundable credit is the $500 other-dependent credit, which leaves $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It brought in itemized mortgage and medical deductions, but no mortgage interest amount was given and medical expenses over the 7.5% floor ($16,621) fall short of the $32,200 standard deduction. Its $4,940 is below the correct $6,116.24 of tax minus the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated the dependent's $10,800 of disability benefits as Social Security and added 85% ($9,180) to the joint return's income. Those benefits are the dependent's own nontaxable disability income and stay out of the couple's AGI, so taxable income is $55,102, not $64,282." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $8,500 overstates the tax. Wages plus pension less the $32,200 standard deduction gives $55,102 of taxable income, which produces $6,116.24 at the 2026 10%/12% rates, and the $500 other-dependent credit brings it to $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA' rules: itemized deductions of medical plus SALT, $15,750 of personal exemptions, and a 15% bracket. OBBBA made the TCJA permanent, so 2026 has a $32,200 standard deduction, no exemptions, and 10%/12% brackets. It also missed the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset and used a $16,600 standard deduction, three $5,300 exemptions, and a 15% bracket. Under OBBBA, 2026 has a $32,200 MFJ standard deduction, no exemptions, and 10%/12% brackets, and the $500 other-dependent credit applies, which the model also left out." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the expired pre-TCJA structure (itemizing, $15,900 of exemptions, 15% bracket) and wrongly said the other-dependent credit had expired. OBBBA made the $500 credit permanent and set the 2026 MFJ standard deduction at $32,200 with 10%/12% brackets." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It indexed a pre-TCJA $17,291 standard deduction and applied a 15% bracket, and it left out personal exemptions even under that framework. The 2026 rules under OBBBA are a $32,200 standard deduction and 10%/12% brackets, with a $500 other-dependent credit that it also didn't apply." -us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It claimed no dependents, so it left out the $500 credit for other dependents. The permanently disabled 23-year-old is a qualifying child, because disability waives the age test, and is eligible for that credit. It also used a $31,500 standard deduction instead of the 2026 figure of $32,200." -us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of OBBBA's $32,200 for 2026 MFJ and ended the 10% bracket at $24,400 instead of $24,800, which overstated taxable income and tax. It applied the $500 other-dependent credit correctly." -us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It made two errors. First, it treated the dependent's disability benefits as Social Security and added $9,180 to AGI, although those benefits are excluded from the joint return. Second, it subtracted a $1,845 auto-loan-interest deduction whose qualifying conditions aren't met. Correct taxable income is $55,102." -us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It gave a $2,000 child tax credit for the 23-year-old, but the CTC requires the child to be under 17, so a disabled adult qualifying child gets only the $500 credit for other dependents. It also added the dependent's nontaxable $10,800 of disability benefits to AGI (and mis-added the total as $97,202) and used a $30,000 standard deduction instead of $32,200." -us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the pre-credit tax of $6,116.24 exactly, then applied no nonrefundable credit, reasoning that a 23-year-old can't get the CTC. It missed the $500 credit for other dependents, which applies because a permanently and totally disabled child of any age is a qualifying child." -us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the dependent's $10,800 of disability benefits as taxable income on the joint return, raising AGI to $98,102 instead of $87,302. It also used a $31,200 standard deduction and a $24,150 10% bracket instead of 2026's $32,200 and $24,800." -us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a made-up $10,000 child tax credit that wiped out the whole liability. The household has no CTC-eligible child under 17, and the only nonrefundable credit is the $500 credit for the disabled adult dependent, which leaves $5,616.24 of tax on $55,102 of taxable income." +us,scenario_067,dependent1_medicare_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model treated being disabled and receiving disability benefits as enough for Medicare. Under-65 Medicare requires 24 months of SSDI receipt or ESRD. The listed benefit is not SSDI, and SSDI months and ESRD are unlisted, so they default to 0 and false." +us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model claimed a PolicyEngine disability rule that is triggered by disability status plus disability benefits. The actual rule requires at least 24 months of Social Security disability receipt (or ESRD). Neither is listed for Dependent 1, so both default to 0/false and the dependent fails the test at age 23." +us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the 24-month SSDI waiting period but assumed it was satisfied because disability status is constant for the year. It also relabeled the generic $10,800 disability benefit as SSDI. Months of SSDI receipt are unlisted and default to 0, so the 24-month test fails." +us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model cited the 24-month SSDI waiting period but then counted Dependent 1 as eligible without any months of SSDI receipt being given. It also assumed the generic disability benefits were SSDI. Unlisted SSDI months default to 0, so the 24-month requirement fails and the dependent is not eligible." +us,scenario_067,dependent1_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model went straight from ""disabled"" to Medicare eligibility. Disability status alone does not qualify anyone under 65. Medicare requires at least 24 months of SSDI receipt or ESRD, and neither is present here." +us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated disability status plus annual disability benefits as support for under-65 Medicare. The disability pathway requires at least 24 months of Social Security disability receipt, not generic disability-benefit income. With SSDI months and ESRD defaulting to 0/false, Dependent 1 fails every pathway." +us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model said PolicyEngine treats a disabled person with disability benefits as Medicare-eligible. PolicyEngine's rule actually requires age 65 or older, ESRD, or 24+ months of SSDI receipt. The generic `disability_benefits` input satisfies none of these, and SSDI months default to 0." +us,scenario_067,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied a disability-based Medicare pathway on the strength of the disabled flag and the reported disability benefits. That pathway is keyed to at least 24 months of Social Security disability receipt, which is unlisted and therefore 0. Dependent 1 does not qualify." +us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model asserted that PolicyEngine treats disability benefits as qualifying for Medicare under age 65. Only 24 or more months of SSDI receipt, or ESRD, qualifies someone under 65. Generic disability-benefit income meets neither, and both default to 0/false here." +us,scenario_067,dependent1_medicare_eligible,muse-wonder,llm_error,age_disability,False,"The model applied the disability Medicare pathway using only the disabled flag and generic disability benefits. That pathway requires at least 24 months of SSDI receipt. SSDI months are unlisted and default to 0, so the 23-year-old is not eligible." +us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model found Medicare eligibility ""under disability rules"" from disability status and disability benefits alone. Those rules require at least 24 months of SSDI receipt or ESRD, and both default to 0/false for this dependent." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working reached $5,347, but it submitted $3,391, a number its reasoning never supports. The $5,347 was also wrong for two reasons. It subtracted the $1,845 auto-loan-interest deduction, which does not apply because no qualifying new US-assembled vehicle loan is given. It also used a $32,600 standard deduction instead of $32,200." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,845 OBBBA auto-loan-interest deduction, but the household facts never establish a qualifying new, US-assembled vehicle loan, so that deduction is $0. This lowered taxable income to $53,257 instead of $55,102 and the tax by exactly $221.40 (12% × $1,845)." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 standard deduction instead of $32,200 and miscomputed the tax on $58,802 as $7,080. It then subtracted about $1,266 of nonrefundable credits it made up, rather than the single $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It explicitly assumed no nonrefundable credits, missing the $500 credit for other dependents for the permanently disabled 23-year-old. Its $6,900 is also about $780 above the correct pre-credit tax of $6,116.24, which means it understated the $32,200 MFJ standard deduction." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own steps produced about $5,620 after the $500 credit for other dependents. It then raised the answer to $6,280 under an unexplained 'precise brackets' adjustment that used stale 2025 bracket limits ($23,850) instead of the 2026 $24,800 limit for the 10% bracket." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It made up about $6,500 of mortgage interest from the $100,000 balance, but unlisted interest is $0. On that basis it wrongly chose to itemize instead of taking the $32,200 standard deduction. It then submitted $5,594, which does not match its own $6,722 computation." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly reached taxable income of $55,102, tax of $6,116, and about $5,616 after the $500 credit for other dependents. It then threw that result away and submitted $6,104 with no supporting arithmetic." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It took the $1,845 auto-loan-interest deduction although no qualifying new, US-assembled vehicle loan is established, so the deduction is $0. This cut taxable income to $53,257 instead of $55,102 and understated the tax by $221.40." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 standard deduction to $30,000 instead of using the 2026 OBBBA amount of $32,200. It also guessed a $24,500 limit for the 10% bracket instead of $24,800. Together these overstated taxable income at $57,302 instead of $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own arithmetic gave $6,068 minus the $500 credit for other dependents, or $5,568, which was already off because it used a $32,600 standard deduction instead of $32,200. It then submitted $10,500, citing made-up 'self-employment-like adjustments' that do not exist for a wage-and-pension household." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 disability benefits to the joint return's AGI ($98,102 instead of $87,302) and used a $30,600 standard deduction instead of $32,200. It also denied the $500 credit for other dependents, although a permanently disabled 23-year-old qualifies as a dependent." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the TCJA sunset, using a $16,500 standard deduction, three personal exemptions, and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 MFJ standard deduction, no exemptions, a 12% bracket, and the $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 law: itemized medical expenses and state tax, three $5,300 personal exemptions, and the 15% bracket. Under OBBBA, the $32,200 standard deduction, 10%/12% brackets, and $500 credit for other dependents apply. That leaves $55,102 of taxable income and a $500 credit, which it omitted." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,845 auto-loan-interest deduction, which does not apply without a qualifying new, US-assembled vehicle loan. That pushed taxable income below the correct $55,102 and produced $5,486.84 instead of $5,616.24." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the TCJA sunset (personal exemptions, lower standard deduction, 15% bracket) even though OBBBA kept the $32,200 MFJ standard deduction and the 10%/12% brackets. It also added 85% of the dependent's $10,800 disability benefits to the joint AGI as taxable Social Security, although those benefits are excluded." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $31,200 standard deduction instead of $32,200 and miscomputed taxable income as $60,400; the correct figure is $55,102. It also stated there were no nonrefundable credits, missing the $500 credit for other dependents for the disabled 23-year-old." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied personal exemptions, itemized deductions, and the 15% bracket. OBBBA made the $32,200 MFJ standard deduction, 10%/12% brackets, and the $500 credit for other dependents permanent." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA rules (personal exemptions and the 15% bracket) instead of the OBBBA-permanent 2026 structure. It also made up $4,500 of mortgage interest from the balance and so itemized instead of taking the $32,200 standard deduction." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The correct derivation gives taxable income of $55,102, tax of $6,116.24, and $5,616.24 after the $500 credit for other dependents. Its $7,534 is $1,918 too high, which matches dropping the $500 credit and either using a much smaller standard deduction or adding the dependent's disability benefits to taxable income." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of the 2026 OBBBA $32,200 and the 2025 limit of $23,850 for the 10% bracket instead of $24,800. This overstated taxable income at $56,502 and the pre-credit tax at $6,303.24." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It got AGI and the $500 credit for other dependents right, but its pre-credit tax of $6,412.24 matches a stale standard deduction of about $30,000 and an outdated 10% bracket limit. The correct 2026 figures are a $32,200 deduction and a $24,800 limit, giving tax of $6,116.24." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA $30,000 standard deduction and the 2025 limit of $23,850 for the 10% bracket. The 2026 values are $32,200 and $24,800, so it overstated taxable income at $57,302 instead of $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction instead of the 2026 OBBBA $32,200, and a limit of $23,600 for the 10% bracket instead of $24,800. This inflated taxable income to $57,302 and the pre-credit tax to $6,404.24." +us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied the $500 credit for other dependents by applying the qualifying-relative gross income test. A permanently and totally disabled child is a qualifying child with no age limit and no gross income test, and the disability benefits are not countable taxable income anyway. It also used a $25,200 limit for the 10% bracket instead of $24,800." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The correct computation is $87,302 AGI minus the $32,200 standard deduction, giving $55,102 taxable, $6,116.24 of tax, and $5,616.24 after the $500 credit for other dependents. Its $4,567 is $1,049 too low, which matches about $8,700 of extra deductions at 12%, or an equivalent over-credit, beyond what the facts support." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It cited itemized deductions based on the mortgage and medical items, but unlisted mortgage interest is $0 and medical expenses above the 7.5% floor ($16,621) fall short of the $32,200 standard deduction. Its $4,940 is $676 below the correct $5,616.24, which matches about $5,600 of deductions it should not have taken." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated $9,180 (85%) of the dependent's $10,800 disability benefits as taxable Social Security on the joint return, raising taxable income to $64,282. Those benefits are excluded, so taxable income is $55,102 and tax after the $500 credit for other dependents is $5,616.24." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $8,500 is about $2,880 above the correct $5,616.24. That matches skipping the $500 credit for other dependents and either using a far smaller deduction than the 2026 $32,200 MFJ standard deduction or taxing the dependent's disability benefits. It gave no computation of the $55,102 taxable income." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA' 2026 rules (personal exemptions, itemizing medical expenses and state tax, 15% bracket) and claimed no nonrefundable credits. OBBBA kept the $32,200 MFJ standard deduction, the 10%/12% brackets, and the $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, using a $16,600 standard deduction, three $5,300 exemptions, and the 15% bracket, and it applied no dependent credit. Under OBBBA, the $32,200 standard deduction and 10%/12% brackets give $6,116.24 of tax, reduced by the $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules (personal exemptions, itemizing, 15% bracket) and said the credit for other dependents had expired. OBBBA made the $32,200 standard deduction and the $500 credit for other dependents permanent, so the disabled 23-year-old earns a $500 credit." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a reverted pre-TCJA standard deduction of $17,291 and the 15% bracket. It also left out the personal exemptions its own sunset model would require and denied the $500 credit for other dependents. The 2026 OBBBA law uses a $32,200 deduction, 10%/12% brackets, and the $500 credit." +us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It claimed no dependents and so omitted the $500 credit for other dependents for the permanently disabled 23-year-old. It also used the 2025 $31,500 standard deduction instead of the 2026 $32,200." +us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used an estimated $30,800 standard deduction and a $24,400 limit for the 10% bracket instead of the 2026 OBBBA values of $32,200 and $24,800. This overstated taxable income at $56,502 instead of $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It added $9,180 of the dependent's disability benefits to the joint AGI as taxable Social Security, although those benefits are excluded. It also subtracted a $1,845 auto-loan-interest deduction that does not apply without a qualifying new, US-assembled vehicle loan." +us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It gave a $2,000 child tax credit for a 23-year-old, but the CTC requires the child to be under 17; a disabled adult child gets only the $500 credit for other dependents. It also included the dependent's $10,800 disability benefits in AGI and used a $30,000 standard deduction instead of $32,200." +us,scenario_067,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It followed the right path ($55,100 taxable, 10%/12% brackets, $500 credit for other dependents) but rounded the pre-credit tax to $6,124. The exact figure is $6,116.24 on $55,102 (2,480 + 12% × 30,302), so its answer is $7.76 too high." +us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the pre-credit tax of $6,116.24 correctly but applied no nonrefundable credits because the dependent is 23. A permanently and totally disabled child is a qualifying dependent at any age, so the $500 nonrefundable credit for other dependents applies, giving $5,616.24." +us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It treated the dependent's $10,800 disability benefits as taxable income on the joint return, giving $98,102 instead of $87,302. It also used a $31,200 standard deduction and a $24,150 limit for the 10% bracket instead of $32,200 and $24,800." +us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a made-up $10,000 child tax credit that wiped out all the tax. The household has no child under 17, so the only dependent credit is the $500 nonrefundable credit for other dependents for the disabled 23-year-old, which leaves $5,616.24." us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability can remove the age limit for the qualifying-child relationship test used for dependency and EITC purposes, but it does not override the CTC requirement that the child be under age 17, so the ACTC is $0." us,scenario_067,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented five qualifying children even though the household contains only one dependent, age 23, and then constructed a $10,000 CTC and $1,210 ACTC from that nonexistent child count. The sole dependent fails the CTC under-age-17 requirement, so there is no CTC remainder to refund and no ACTC." us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 60 as above the Medicare eligibility threshold. The applicable age threshold is 65, and no disability-based Medicare pathway or other exception applies to the head, so the correct value is 0." -us,scenario_067,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model read the prompt's list of four city taxes as complete and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: Adams County's default 1.6% rate on Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) gives $1,348.83." -us,scenario_067,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model treated the NYC/Philadelphia/Kansas City/St. Louis list as complete and answered $0. It never applied the Indiana county income tax that PolicyEngine includes in local_income_tax, which is 1.6% (Adams County default) of $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model said Indiana residents owe no local income tax. In fact, every Indiana county levies a county income tax on Indiana AGI. PolicyEngine includes that tax in local_income_tax: 1.6% (Adams County default) × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model correctly saw that Indiana county income tax applies, but $1,800 is an unexplained round number equal to about 2.1% of income. The engine applies Adams County's 1.6% rate to Indiana AGI of $84,302 ($87,302 federal AGI minus $3,000 of exemptions for three people), which gives $1,348.83." -us,scenario_067,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed city taxes and answered $0. It left out the Indiana county income tax that PolicyEngine's local_income_tax includes: 1.6% of $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model treated the output as covering only NYC, Philadelphia, Kansas City and St. Louis taxes, so it answered $0. PolicyEngine's local_income_tax also includes Indiana county tax, computed at the Adams County default rate of 1.6% on $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model knew Indiana county income tax exists but decided the prompt's city list excluded it from this output, so it answered $0. PolicyEngine includes the county tax in local_income_tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It skipped the Indiana county income tax in PolicyEngine's local_income_tax, which is 1.6% (Adams County default) of $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model read the four named jurisdictions as the only taxes this output covers and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax at 1.6% of $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model treated the enumerated city taxes as complete and answered $0. It missed the Indiana county income tax that PolicyEngine includes in local_income_tax: Adams County's 1.6% rate × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to the listed NYC/Philadelphia/Kansas City/St. Louis taxes and answered $0. PolicyEngine also counts Indiana county income tax here: 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model treated the four covered city taxes as the only ones in this output and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% (Adams County default) of $84,302 = $1,348.83." -us,scenario_067,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It never applied the Indiana county income tax that PolicyEngine includes in local_income_tax, 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model read the listed city taxes as complete and answered $0. PolicyEngine's local_income_tax also covers Indiana county income tax, which comes to $1,348.83 at Adams County's 1.6% rate on $84,302 Indiana AGI." -us,scenario_067,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model answered $0 under the reading that only the enumerated city taxes count. It left out the Indiana county income tax in PolicyEngine's local_income_tax: 1.6% of Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) = $1,348.83." -us,scenario_067,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model read the prompt as covering only NYC, Philadelphia, Kansas City and St. Louis, so it answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model said living in Indiana triggers no local income tax. In fact, all Indiana counties levy a county income tax on Indiana AGI, and PolicyEngine counts it in local_income_tax: Adams County's 1.6% rate × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model recognized Indiana county taxes but decided this output variable excluded them, so it answered $0. PolicyEngine's local_income_tax includes the county tax: 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model treated the four specified city taxes as the full scope of the output and answered $0. It omitted Indiana county income tax, which PolicyEngine computes as 1.6% of $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana's local taxes are not the type this output models and answered $0. PolicyEngine does model Indiana county income tax in local_income_tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model limited the output to the listed NYC/Philadelphia/Kansas City/St. Louis jurisdictions and answered $0. PolicyEngine also includes Indiana county income tax, computed with the Adams County default rate of 1.6% on $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model concluded the household was outside any recognized separate local-tax jurisdiction and answered $0. Indiana county income tax is part of PolicyEngine's local_income_tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model said PolicyEngine's local income tax covers only NYC, Philadelphia, Kansas City and St. Louis, so it answered $0. The engine's local_income_tax also includes Indiana county income tax: 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the complete list and answered $0. It left out the Indiana county income tax in PolicyEngine's local_income_tax: 1.6% of $84,302 = $1,348.83." -us,scenario_067,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model counted only the listed city taxes and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: Adams County's 1.6% rate × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Because no local jurisdiction was named, the model answered $0. PolicyEngine applies its default Indiana county (Adams, 1.6%) to Indiana AGI of $84,302, which gives $1,348.83 of county income tax in local_income_tax." -us,scenario_067,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model checked only for NYC/Philadelphia/Kansas City/St. Louis taxes and answered $0. It skipped the Indiana county income tax that PolicyEngine includes, 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model read the four enumerated city taxes as the full scope and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four listed jurisdictions and answered $0. It left out the Indiana county income tax in PolicyEngine's local_income_tax, which is the Adams County default rate of 1.6% on $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model treated the separately requested city taxes as the complete list and answered $0. PolicyEngine also counts Indiana county income tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model ruled Indiana out because it is not one of the specified jurisdictions and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% of $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana is not covered by this output and answered $0. PolicyEngine includes Indiana county income tax in local_income_tax: 1.6% (Adams County default) × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Because no listed city jurisdiction applied, the model answered $0. It omitted the Indiana county income tax that PolicyEngine includes, 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model treated the listed jurisdictions as complete and answered $0. PolicyEngine's local_income_tax also covers Indiana county income tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"The model said the output covers only the specified city taxes and answered $0. PolicyEngine includes Indiana county income tax in local_income_tax: Adams County's 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model said no local income taxes apply in Indiana. In fact, every Indiana county imposes a county income tax on Indiana AGI. PolicyEngine includes it in local_income_tax at 1.6% (Adams County default) × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model limited the output to the four named city taxes and answered $0. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model recognized Indiana county tax but treated it as outside this output because of the city list, so it answered $0. PolicyEngine counts it in local_income_tax: 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and answered $0. It left out the Indiana county income tax in PolicyEngine's local_income_tax, which is 1.6% of $84,302 = $1,348.83." -us,scenario_067,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model acknowledged that Indiana has county taxes but excluded them because of the prompt's city list, so it answered $0. PolicyEngine includes the county tax: Adams County's 1.6% rate × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model treated the four listed city taxes as the full scope and answered $0. PolicyEngine's local_income_tax includes Indiana county income tax: 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"The model read the output as limited to NYC, Philadelphia, Kansas City and St. Louis taxes and answered $0. PolicyEngine also includes Indiana county income tax, computed as 1.6% of Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) = $1,348.83." -us,scenario_067,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"The model counted only the listed city taxes and answered $0. It skipped the Indiana county income tax in PolicyEngine's local_income_tax, 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model recognized that Indiana county income tax applies but answered $0 because no county was given. PolicyEngine handles a missing county by applying its default, Adams County at 1.6%, to $84,302 Indiana AGI, which gives $1,348.83." -us,scenario_067,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model explicitly decided that Indiana county taxes fall outside this output and answered $0. PolicyEngine's local_income_tax includes them: 1.6% × $84,302 Indiana AGI = $1,348.83." -us,scenario_067,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the only modeled local taxes and answered $0. PolicyEngine also models Indiana county income tax: Adams County's 1.6% × $84,302 = $1,348.83." -us,scenario_067,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model said local taxes apply only in the listed cities and answered $0. It omitted the Indiana county income tax that PolicyEngine includes in local_income_tax, 1.6% of $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model took the four listed city taxes as the full scope of local_income_tax and left out Indiana county income tax, which PolicyEngine counts in this output. With the default Adams County rate of 1.6% on Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions), the answer is $1,348.83." +us,scenario_067,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model read the NYC/Philadelphia/KC/St. Louis list as exhaustive and left out Indiana county income tax, which is part of PolicyEngine's local_income_tax. That tax is 1.6% (Adams County default) × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model claimed that Indiana does not levy local income taxes. In fact, every Indiana county levies a county income tax on Indiana AGI. PolicyEngine includes that tax in local_income_tax and computes 1.6% (Adams County default) × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model correctly saw that Indiana county income tax applies, but its $1,800 implies a rate of about 2.1% on an unstated base. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $84,302 ($87,302 federal AGI minus $3,000 of exemptions for three people), which gives $1,348.83." +us,scenario_067,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the only parts of local_income_tax and so left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model read the output as limited to NYC, Philadelphia, KC and St. Louis taxes, but PolicyEngine's local_income_tax also includes Indiana county income tax. Using the default Adams County rate of 1.6% on $84,302 of Indiana AGI, that tax is $1,348.83." +us,scenario_067,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model knew about Indiana county income tax but left it out because the prompt's list names only four city taxes. PolicyEngine includes the county tax in local_income_tax: 1.6% (Adams County) × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model checked only the four listed city taxes and never applied Indiana county income tax, which PolicyEngine counts in local_income_tax. That tax is 1.6% × ($87,302 − $3,000 exemptions) = $1,348.83." +us,scenario_067,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model treated the four named jurisdictions as an exhaustive list and so left out Indiana county income tax. PolicyEngine includes that tax in local_income_tax at the default Adams County rate of 1.6% on $84,302 of Indiana AGI, giving $1,348.83." +us,scenario_067,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model read the prompt's list of city taxes as the whole definition and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana has no tax within the listed outputs, so it never applied the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County) on $84,302 of Indiana AGI, or $1,348.83." +us,scenario_067,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"The model limited local_income_tax to the four named city taxes and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model checked only the NYC/Philadelphia/KC/St. Louis taxes and never applied Indiana county income tax, which is part of PolicyEngine's local_income_tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model treated the listed city taxes as exhaustive and left out Indiana county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on Indiana AGI of $84,302, giving $1,348.83." +us,scenario_067,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model found no applicable local tax because it followed the prompt's city-only list, but PolicyEngine's local_income_tax includes Indiana county income tax. That tax is 1.6% × ($87,302 − $3,000) = $1,348.83." +us,scenario_067,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model read the output as covering only NYC, Philadelphia, KC and St. Louis, so it left out Indiana county income tax. PolicyEngine includes that tax at 1.6% (Adams County) on $84,302, giving $1,348.83." +us,scenario_067,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model claimed that an Indiana residence triggers no local income tax. In fact, every Indiana county levies a county income tax, and PolicyEngine reports it in local_income_tax as 1.6% × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model recognized Indiana county tax but left it out of this output because of the prompt's city-only description. PolicyEngine's local_income_tax includes the county tax: 1.6% (Adams County) × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model treated the four specified city taxes as the full scope of the output and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana's local taxes fall outside the modeled output, but PolicyEngine's local_income_tax includes Indiana county income tax. That tax is 1.6% on $84,302 of Indiana AGI, or $1,348.83." +us,scenario_067,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model limited local_income_tax to the named city jurisdictions and left out Indiana county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on Indiana AGI of $84,302, giving $1,348.83." +us,scenario_067,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model found no recognized separate local tax because it followed the prompt's city list, but PolicyEngine's local_income_tax includes Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model claimed that PolicyEngine's local income tax covers only NYC, Philadelphia, KC and St. Louis. In fact the variable also includes Indiana county income tax, which is 1.6% (Adams County default) × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"The model read the listed city taxes as exhaustive and left out Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% on $84,302 of Indiana AGI, or $1,348.83." +us,scenario_067,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model treated the four named taxes as the only components and so left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × ($87,302 − $3,000 exemptions) = $1,348.83." +us,scenario_067,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model applied no local tax because no city jurisdiction was named, but PolicyEngine applies Indiana county income tax using the default Adams County rate of 1.6%. On Indiana AGI of $84,302, that gives $1,348.83." +us,scenario_067,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC/Philadelphia/KC/St. Louis taxes and left out Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the full definition and left out Indiana county income tax. PolicyEngine includes that tax at 1.6% (Adams County) on $84,302 of Indiana AGI, giving $1,348.83." +us,scenario_067,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model checked only the listed city jurisdictions and never applied Indiana county income tax, which is part of PolicyEngine's local_income_tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model read the output as limited to the four named city taxes and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × ($87,302 − $3,000) = $1,348.83." +us,scenario_067,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model treated Indiana as outside the specified jurisdictions, but PolicyEngine's local_income_tax includes Indiana county income tax. That tax is 1.6% (Adams County default) on $84,302 of Indiana AGI, or $1,348.83." +us,scenario_067,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model concluded that Indiana is not covered by the output, but PolicyEngine's local_income_tax includes Indiana county income tax. That tax is 1.6% × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"The model followed the prompt's city-only categories and left out Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default, since no county was given) × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model treated Indiana as outside the output's jurisdictions and left out Indiana county income tax. PolicyEngine includes that tax at 1.6% on Indiana AGI of $84,302, giving $1,348.83." +us,scenario_067,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"The model limited the output to the specified city taxes and left out Indiana county income tax, which PolicyEngine counts in local_income_tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model claimed that no local income taxes apply in Indiana. In fact, all Indiana counties levy a county income tax, and PolicyEngine computes it as 1.6% (Adams County) × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model treated the four named city taxes as the only things that feed into local_income_tax and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model recognized Indiana county tax but placed it outside this output because of the prompt's city list. PolicyEngine includes it in local_income_tax at 1.6% on $84,302, giving $1,348.83." +us,scenario_067,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model limited the output to NYC, Philadelphia, KC and St. Louis and left out Indiana county income tax. PolicyEngine includes that tax at the default Adams County rate of 1.6% on Indiana AGI of $84,302, giving $1,348.83." +us,scenario_067,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model acknowledged Indiana county taxes but left them out because the output description lists only city taxes. PolicyEngine's local_income_tax includes the county tax: 1.6% × ($87,302 − $3,000) = $1,348.83." +us,scenario_067,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model read the listed city taxes as exhaustive and never applied Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"The model treated the four specific city taxes as the whole scope of the output and left out Indiana county income tax. PolicyEngine includes that tax at 1.6% (Adams County) on $84,302 of Indiana AGI, giving $1,348.83." +us,scenario_067,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"The model checked only the NYC/Philadelphia/KC/St. Louis taxes and left out Indiana county income tax, which is part of PolicyEngine's local_income_tax. That tax is 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model set the tax to $0 because no Indiana county was given. When no county is given, PolicyEngine defaults to Adams County and applies its 1.6% rate to Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions), giving $1,348.83." +us,scenario_067,local_income_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"The model limited local_income_tax to the four named cities and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × $84,302 = $1,348.83." +us,scenario_067,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model explicitly placed Indiana county taxes outside this output because of the prompt's city list. PolicyEngine's local_income_tax includes them: 1.6% (Adams County default) × $84,302 Indiana AGI = $1,348.83." +us,scenario_067,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model treated the four modeled city taxes as exhaustive and left out Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% on $84,302 of Indiana AGI, or $1,348.83." +us,scenario_067,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model read the listed jurisdictions as the only ones with local income tax and left out Indiana county income tax. PolicyEngine includes that tax and computes 1.6% × ($87,302 − $3,000) = $1,348.83." us,scenario_067,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model first got the correct $6,300.54 from 7.65% of the head's $82,360. It then made up spouse wages by applying the head's $67 hourly rate to the spouse's 50 weekly hours, even though the prompt says unlisted numbers are 0. After trying several inconsistent versions of that guess, it submitted an unexplained $12,554.41 instead of the head-only figure." us,scenario_067,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model assumed the spouse had wages that the prompt does not list, even though the spouse's wages are $0. At 7.65%, its $8,500 answer implies about $111,000 of taxed wages, roughly $28,700 more than the head's $82,360. Taxing only the head's wages gives $6,300.54." us,scenario_067,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the right base and rates but rounded each piece to whole dollars ($5,106 and $1,194). It then added them to $6,299 instead of $6,300 and blamed a wage limit that does not apply at $82,360. The unrounded total is $5,106.32 + $1,194.22 = $6,300.54." @@ -5669,23 +5732,24 @@ us,scenario_067,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,Fal us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly said payroll tax applies only to the head's $82,360 of wages and that no Additional Medicare Tax is due. Its $5,727.14 is still $573.40 short of the correct $5,106.32 + $1,194.22 = $6,300.54. That figure equals 7.65% of only about $74,864 of wages, which shows an arithmetic error in combining the Social Security and Medicare pieces." us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated $5,106 Social Security and $1,194 Medicare on the head's $82,360. It then described the total as a range of $6,300 to $6,400 and arbitrarily submitted $6,376 instead of adding the two pieces to get $6,300.54." us,scenario_067,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as categorically SSI-eligible on disability and computed (994 − 880) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, as it does for the head (60) and spouse (58), so no member passes the aged/blind/disabled gate and SSI is $0 before the income test." -us,scenario_067,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the 23-year-old as an SSI-disabled adult with no parental deeming and paid (994 − 880) × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and for both adults, so SSI's categorical test fails for every member and the benefit is $0." -us,scenario_067,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Assumed Dependent 1 meets SSI's disability criterion and netted $10,560 of countable unearned income against an $11,928 annual FBR to get $1,368. The engine's is_ssi_aged_blind_disabled=False for Dependent 1 (and for the head and spouse, both under 65) stops SSI at the categorical step, giving $0." -us,scenario_067,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability and subtracted $10,560 of countable income from an understated 2026 FBR of about $991/month, getting $1,334. The engine sets is_ssi_aged_blind_disabled=False for every member, so SSI is $0 and the FBR and income steps never apply." -us,scenario_067,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as SSI-disabled and computed (991 − 880) × 12 = $1,332, using an FBR below the 2026 rate of $994. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, the head (60) and the spouse (58), so no one clears SSI's categorical gate and SSI is $0." -us,scenario_067,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as a disabled adult eligible for SSI and computed $11,889 − $10,560 = $1,329 from an assumed $990.75/month FBR. The engine evaluates is_ssi_aged_blind_disabled=False for all three members, so SSI is $0 before any unearned-income offset." -us,scenario_067,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. Separately, it used the 2025 FBR of $967/month instead of the 2026 rate and subtracted the $20 general exclusion once a year instead of monthly ($240/year), which gives $11,604 − $10,780 = $824." -us,scenario_067,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Declared the disabled 23-year-old the only SSI-eligible member and paid $114/month × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 too, so no member meets SSI's categorical requirement and SSI is $0." -us,scenario_067,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and netted $10,560 of countable disability income against an $11,928 FBR to get $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 and both adults, so SSI stops at the categorical test with $0." -us,scenario_067,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Treated the disabled adult dependent as categorically SSI-eligible with no parental deeming and computed $11,928 − $10,560 = $1,368. The engine's is_ssi_aged_blind_disabled is False for every member, so the aged/blind/disabled gate yields $0 SSI." -us,scenario_067,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as SSI-disabled and paid (994 − 880) = $114/month, or $1,368 a year. The engine evaluates is_ssi_aged_blind_disabled=False for Dependent 1, the head (60) and the spouse (58), so SSI is $0 before the income offset." -us,scenario_067,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability and reduced the $994 FBR by $880 of countable income to get $114/month ($1,368/year). The engine's is_ssi_aged_blind_disabled is False for all members, so no one passes SSI's categorical test and the benefit is $0." -us,scenario_067,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Named Dependent 1 the only categorically eligible member and computed $114/month × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 as well as for the head and spouse, so SSI is $0 at the eligibility gate." -us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. Separately, it subtracted the $20 general exclusion once a year instead of monthly ($240/year) and used an $11,892 annual FBR, which gives $11,892 − $10,780 = $1,112." -us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and computed ($994 − $880) × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and both adults, so SSI's categorical requirement fails for everyone and SSI is $0." -us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as an SSI-disabled adult with no parental deeming and paid (994 − 880) × 12 = $1,368. The engine evaluates is_ssi_aged_blind_disabled=False for every member, so SSI is $0 before any income or deeming step." -us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. On top of that, it used an invented $1,913/month FBR, well above the 2026 individual rate of $994 and the couple rate of about $1,491, and skipped the $20 general exclusion, which gives $22,956 − $10,800 = $12,156." +us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated Dependent 1's 'is disabled' flag as meeting the SSI disability test and computed (994 − 880) × 12 = $1,368. Under the engine's inputs, Dependent 1's is_ssi_aged_blind_disabled is False, and the head (60) and spouse (58) are under 65. No member is categorically eligible, so SSI is $0." +us,scenario_067,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the disabled 23-year-old as SSI-eligible and computed (994 − 880) × 12 = $1,368. The engine gives Dependent 1 is_ssi_aged_blind_disabled=False, and neither parent is 65 or older. No member passes the categorical test, so SSI is $0." +us,scenario_067,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Assumed Dependent 1 qualifies as SSI-disabled and subtracted $10,560 of countable disability income from the $11,928 annual FBR, giving $1,368. The engine's inputs set is_ssi_aged_blind_disabled=False for Dependent 1, and the head and spouse are under 65. The benefit formula never applies, and SSI is $0." +us,scenario_067,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as categorically SSI-disabled and ran the benefit formula, also understating the 2026 FBR as about $991/month instead of $994, which gave $1,334. The engine gives Dependent 1 is_ssi_aged_blind_disabled=False, and the head and spouse are under 65, so SSI is $0." +us,scenario_067,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Awarded SSI to Dependent 1 as a disabled individual, using a $991 monthly FBR instead of the 2026 rate of $994, which gave (991 − 880) × 12 = $1,332. The engine's SSI aged/blind/disabled status is False for Dependent 1 and for both parents (ages 60 and 58), so SSI is $0." +us,scenario_067,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as an SSI-eligible disabled adult and used a $990.75 monthly FBR instead of $994, which gave $1,329. Under the engine's inputs, Dependent 1's is_ssi_aged_blind_disabled is False, and the head and spouse are under 65. No member qualifies, so SSI is $0." +us,scenario_067,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Granted SSI to Dependent 1 even though the engine sets is_ssi_aged_blind_disabled=False for every member, so SSI is $0. The arithmetic was also wrong: the model used the 2025 FBR of $967/month instead of the 2026 rate of $994, and took the $20 general exclusion once a year instead of every month ($240/year). That produced $824." +us,scenario_067,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Stated that only the disabled 23-year-old is SSI-eligible and computed $114/month × 12 = $1,368. The engine gives Dependent 1 is_ssi_aged_blind_disabled=False, and the head (60) and spouse (58) are not aged. No one in the household meets the categorical test, so SSI is $0." +us,scenario_067,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Applied the $11,928 annual FBR minus $10,560 of countable disability income to reach $1,368, which assumes Dependent 1 is SSI-disabled. The engine's inputs set is_ssi_aged_blind_disabled=False for Dependent 1 and for both under-65 parents, so SSI is $0." +us,scenario_067,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Treated the disabled adult dependent as categorically SSI-eligible and computed $11,928 − $10,560 = $1,368. Under the engine's inputs, Dependent 1's is_ssi_aged_blind_disabled is False, and neither parent is 65 or older, so SSI is $0." +us,scenario_067,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Computed $114/month for the disabled adult on the assumption that Dependent 1 meets SSI's disability criterion. The engine gives Dependent 1 is_ssi_aged_blind_disabled=False, and the head and spouse are below 65. The benefit computation never runs, and SSI is $0." +us,scenario_067,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Reduced the $994 FBR by $880 of countable income to get $114/month ($1,368/year), which assumes Dependent 1 is SSI-disabled. The engine evaluates Dependent 1 with is_ssi_aged_blind_disabled=False, and both parents are under 65, so SSI is $0." +us,scenario_067,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Declared Dependent 1 the only categorically eligible member and computed $114/month, or $1,368/year. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 as well as for the 60-year-old head and the 58-year-old spouse. No member is eligible, so SSI is $0." +us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Granted SSI to Dependent 1 even though the engine sets is_ssi_aged_blind_disabled=False for every member, so SSI is $0. The arithmetic was also wrong: the model projected an $11,892 annual FBR instead of $11,928 ($994 × 12), and subtracted the $20 general exclusion only once a year instead of monthly. That produced $1,112." +us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and computed ($994 − $880) × 12 = $1,368. Under the engine's inputs, Dependent 1's is_ssi_aged_blind_disabled is False, and the head and spouse are under 65. No member is categorically eligible, so SSI is $0." +us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Assumed the disabled adult dependent meets SSI's disability criterion and computed $114/month × 12 = $1,368. The engine gives Dependent 1 is_ssi_aged_blind_disabled=False, and neither parent is aged, so SSI is $0." +us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Granted SSI to Dependent 1 even though the engine sets is_ssi_aged_blind_disabled=False for every member, so SSI is $0. The arithmetic was also badly wrong: the model used a nonexistent $1,913/month FBR (the 2026 individual rate is $994) and skipped the $20 general income exclusion. That produced $12,156." +us,scenario_067,ssi,muse-wonder,prompt_ambiguity,age_disability,False,"Treated the disabled dependent as SSI-eligible and computed ($994 − $880) × 12 = $1,368. The engine evaluates Dependent 1 with is_ssi_aged_blind_disabled=False, and the head and spouse are under 65, so SSI is $0." us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model invented a $500 disabled-dependent exemption, giving $3,500 instead of $3,000. It applied a ~2.9% rate instead of the 2026 rate of 2.95%, then cut the result to $2,295 with unexplained 'minor deductions'. The correct figure is $84,302 × 2.95% = $2,486.91." us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the federal $28,700 standard deduction even though Indiana starts from AGI and has no standard deduction. It used a 3.15% rate and invented health-premium, medical and senior credits. Its final $4,168 contradicts its own interim $1,356. Indiana allows only $3,000 of exemptions and taxes the remaining $84,302 at 2.95%." us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model stated a 3.05% rate, which is the 2024 rate rather than the 2026 rate of 2.95%. Its $4,500 answer works out to about 5.3% of the ~$84,302 post-exemption base, so it did not even apply the rate it stated. The correct tax is $84,302 × 2.95% = $2,486.91." @@ -5726,41 +5790,42 @@ us,scenario_067,state_income_tax_before_refundable_credits,ox-alpha,llm_error,ta us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model got the $84,302 base right but used 3.05% instead of 2.95%. It then subtracted an invented $100 dependent credit and $374 of made-up 'estimated nonrefundable credits'. No nonrefundable Indiana credits apply." us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model added the dependent's $10,800 disability benefits to the filers' AGI. The correct AGI is $87,302 (wages plus pension). It also used 3.05% instead of the 2026 rate of 2.95%." us,scenario_067,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of the educational assistance (the amount above the §127 cap) to taxable income, but the assistance is excluded entirely and AGI is wages alone. It then submitted $8,258, which does not match its own $13,573 calculation. The correct figure is $17,983.53 taxed at 10%/12%, which is $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It used the outdated $14,600 standard deduction and subtracted the $5,250 educational exclusion from wages. It then let the EITC wipe out the tax, but the EITC is refundable and does not reduce tax before refundable credits. With wage-only AGI, the $16,100 deduction leaves $17,983.53 of taxable income and $1,910.02 of tax." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"Its reasoning reached roughly $18,000 of taxable income, a positive tax, and no nonrefundable credits, yet it submitted $0. Carrying the $17,983.53 of taxable income through the 10%/12% brackets gives $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated $67,750 of the educational assistance as taxable wages, raising AGI to $101,834 and pushing income into the 22% bracket. It also used wrong 2026 bracket edges ($12,150/$49,350). The assistance is excluded entirely, so taxable income is $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added $67,750 of the educational assistance (above the §127 cap) to AGI. The assistance is excluded entirely, so AGI is $34,083.53 and taxable income is $17,983.53, which never reaches the 22% bracket." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning used the correct framework: wages only, the $16,100 deduction, $17,984 of taxable income, and a 10% bracket up to $12,400. That gives about $1,910, but it then submitted an unexplained $2,372 instead of carrying that figure through." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included $67,750 of the educational assistance in AGI, which should be wages only. It also used a stale $15,750 standard deduction and $12,200/$46,600 bracket edges in place of the 2026 values ($16,100; $12,400/$50,400)." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It eventually reached the correct $17,984 of taxable income but used a 10% bracket top of $12,150 instead of $12,400, giving $1,915. It then submitted an arbitrary $1,750 instead of the $1,910.02 that the correct brackets produce." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the full $73,000 of educational assistance as taxable income (AGI $107,084), but the assistance is excluded and AGI is $34,083.53. It also used a $15,700 standard deduction instead of the 2026 single amount of $16,100." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 of ESI premiums from the reported gross wages and added $67,750 of educational assistance to AGI. Both adjustments are wrong: AGI is just the $34,083.53 in wages. It also used a $15,400 standard deduction in place of $16,100." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to AGI and applied pre-TCJA law: a $7,900 standard deduction, a $5,050 personal exemption, and 15%/25% rates. Under 2026 law, the $16,100 standard deduction and the 10%/12% brackets apply to wage-only taxable income of $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using a $7,600 standard deduction plus a $5,100 personal exemption and a 15% second bracket. In fact, 2026 has a $16,100 standard deduction, no personal exemption, and a 12% second bracket. It also wrongly subtracted the $8,389 of ESI premiums from the stated gross wages." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It applied 10% to all of its $18,884 of taxable income, ignoring the 12% bracket above $12,400. It then subtracted a nonexistent Child Tax Credit, although the filer has no dependents and no nonrefundable credits. With the $16,100 deduction, tax is $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters, about $14,250 in standard deduction plus personal exemption, instead of the 2026 $16,100 standard deduction. It also subtracted the $8,389 of ESI premiums from gross wages, which understated taxable income at $11,445 instead of $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to AGI and subtracted the ESI premiums, then applied pre-TCJA law ($8,300 deduction, $5,050 exemption, 15%/25% rates). The correct AGI is wages alone, taxed after the $16,100 deduction at 10%/12%." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave $0 with no supporting derivation. Wage-only AGI of $34,083.53 minus the $16,100 standard deduction leaves $17,983.53 of taxable income, which produces $1,910.02 of tax with no offsetting nonrefundable credits." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026, using an $8,300 standard deduction, a $5,100 personal exemption, and a 15% second bracket. Current 2026 law instead has a $16,100 standard deduction, no exemption, and a 12% rate above $12,400." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $11,116 implies taxable income of about $75,000, which only happens if most of the $73,000 educational assistance is added to income. The assistance is excluded, so taxable income is $17,983.53 and tax is $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used stale parameters, a $15,000 standard deduction and an $11,925 top for the 10% bracket, instead of the 2026 values of $16,100 and $12,400. As a result, it overstated taxable income ($19,084 instead of $17,983.53) and pushed more of it into the 12% bracket." -us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to taxable wages, but the assistance is excluded. It also used 2024 parameters (a $14,600 deduction and $11,600/$47,150 brackets) instead of the 2026 values ($16,100; $12,400/$50,400)." -us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It correctly reached $17,984 of taxable income but set the top of the 2026 single 10% bracket at $12,200 instead of $12,400. That taxes $200 too much at 12% and gives $1,914 instead of $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the income is too low to owe tax after the standard deduction, but $34,083.53 of wages exceeds the $16,100 deduction by $17,983.53. That amount is taxed at 10%/12%, giving $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It said tax after the standard deduction was negative or zero, but wages of $34,083.53 minus the $16,100 deduction leave $17,983.53 of positive taxable income. That produces $1,910.02 of tax with no nonrefundable credits to offset it." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, using a combined standard deduction and personal exemption of about $13,650 and a 15% second bracket. The 2026 parameters are a $16,100 standard deduction, no exemption, and a 12% rate above $12,400." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance (above the $5,250 exclusion) in taxable income, raising AGI to $101,834. The assistance is excluded, so AGI is the $34,083.53 in wages and taxable income is $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, producing $107,084 of income. The assistance is excluded, so only the $34,083.53 in wages less the $16,100 deduction is taxable." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $19,000 implies a standard deduction of about $15,000 instead of the 2026 single amount of $16,100. Its bracket edges are also pre-2026, which overstates both taxable income and the amount taxed at 12%." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 2025-style parameters, a $15,000 standard deduction and an $11,925 top for the 10% bracket. The 2026 values are $16,100 and $12,400, which give $17,983.53 of taxable income and $1,910.02 of tax." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: an $8,450 standard deduction, a $5,300 personal exemption, and a 15% second bracket. 2026 law instead has a $16,100 standard deduction, no exemption, and a 12% rate above $12,400." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly reached $17,984 of taxable income but used $12,200 as the top of the 2026 single 10% bracket instead of $12,400. That overstates tax by $4, giving $1,914.08 instead of $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,527 standard deduction and a 15% second bracket instead of the 2026 $16,100 deduction and 12% rate. It also wrongly subtracted the $8,389 of ESI premiums from the stated gross wages." -us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for federal_income_tax_before_refundable_credits. -us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,750 instead of the 2026 amount of $16,100, and set the 10% bracket top at $12,200 instead of $12,400. Both errors overstated taxable income and the amount taxed at 12%." -us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and a $12,150 top for the 10% bracket instead of the 2026 single values of $16,100 and $12,400. That produced $18,334 of taxable income instead of $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It subtracted the EITC to drive tax to $0, but the EITC is refundable and is excluded from tax before refundable credits. In any case, a filer with $34,084 of wages and no children gets no EITC. Wages minus the $16,100 deduction leave $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to wages, which pushed taxable income to $85,734 and the computed tax to $13,573. It then submitted an unrelated $8,258. Educational assistance is not part of AGI, so taxable income is $17,983.53 and tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It offset the tax with the EITC and other speculative credits, but the EITC is refundable and is not subtracted from tax before refundable credits. It also used the outdated $14,600 standard deduction and included $5,250 of educational assistance in AGI. The correct figure is 10%/12% tax on $17,983.53 of taxable income, or $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It found taxable income of about $18,000 with no nonrefundable credits, but submitted $0 instead of the bracket tax. 10% of $12,400 plus 12% of $5,583.53 gives $1,910.02. It also understated that tax as about $1,000." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It counted $67,750 of educational assistance above the §127 limit as wages, which raised AGI to $101,834 and moved income into the 22% bracket. Educational assistance is excluded from AGI, so AGI is just the $34,084 of wages, giving $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used the correct 2026 deduction and brackets but added $67,750 of educational assistance to AGI, which produced $85,734 of taxable income. With educational assistance excluded, taxable income is $17,983.53 and tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly excluded educational assistance, used the $16,100 deduction and the $12,400 10% bracket, and reached about $1,910. It then submitted $2,372 with no supporting computation. The correct value is $1,240 + 12% × $5,583.53 = $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to AGI, used a $15,750 standard deduction instead of the 2026 figure of $16,100, and used 12%/22% bracket breakpoints of $46,600. Excluding educational assistance gives $17,983.53 of taxable income, all of it in the 10% and 12% brackets." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It eventually used wages-only taxable income of $17,984, but it applied a $12,150 10% bracket (getting $1,915) and then submitted an arbitrary $1,750. With the 2026 $12,400 10% bracket, tax is $1,240 + $670.02 = $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It treated the full $73,000 of educational assistance as taxable, which raised AGI to $107,084, and it used a $15,700 standard deduction. Educational assistance is excluded from AGI, and with the $16,100 deduction taxable income is $17,983.53, giving $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 of ESI premiums from the stated gross wages and added $67,750 of educational assistance, which gave an AGI of $93,445. It also used a $15,400 deduction. AGI is simply the $34,083.53 of wages, which gives $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law, with a $7,900 standard deduction, a $5,050 personal exemption, and 15%/25% brackets, and it added $67,750 of educational assistance to AGI. For 2026 the deduction is $16,100 with no personal exemption under the 10%/12% brackets, and educational assistance is excluded, giving $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, using a $7,600 deduction, a $5,100 exemption and a 15% bracket. It also subtracted the $8,389 of ESI premiums from the stated gross wages. For 2026 the $16,100 deduction applies to full wages of $34,083.53, giving $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It subtracted a Child Tax Credit and other nonrefundable credits even though the filer has no children and qualifies for none. It also taxed all income at a flat 10% and used a $15,200 deduction. The correct value is 10%/12% bracket tax on $17,983.53 of taxable income, or $1,910.02, with no credits." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules, with a combined deduction and exemption of $14,250, and it subtracted the $8,389 of ESI premiums from the stated gross wages. That gave $11,445 taxed at 10%. For 2026 the $16,100 deduction applies to wages of $34,083.53, giving $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to AGI, subtracted the ESI premiums, and applied pre-TCJA rules: an $8,300 deduction, a $5,050 exemption and 15%/25% brackets. Educational assistance is excluded and the 2026 OBBBA rules apply, giving $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $0 with no computation, which treats the filer as having no tax liability. Wages of $34,083.53 minus the $16,100 standard deduction leave $17,983.53 taxable, and the 10%/12% brackets produce $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, using an $8,300 deduction, a $5,100 personal exemption and a 15% bracket. For 2026 the single standard deduction is $16,100 with no personal exemption under the 10%/12% brackets, so tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $11,116 answer is consistent with adding the educational assistance to taxable income and taxing it in the 22% bracket. Educational assistance is excluded from AGI, so taxable income is $34,083.53 − $16,100 = $17,983.53 and tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used 2025 parameters: a $15,000 standard deduction and a $11,925 10% bracket. The 2026 figures are a $16,100 deduction and a $12,400 10% bracket, which give $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to wages and used 2024 parameters: a $14,600 deduction and 2024 brackets. Educational assistance is excluded and the 2026 deduction is $16,100, giving $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It ended the 10% bracket at $12,200 instead of the 2026 single threshold of $12,400, which moved $200 from the 10% rate to 12%. The correct tax is $1,240 + 12% × $5,583.53 = $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It claimed $34,084 of wages is too little to owe tax after the standard deduction. The $16,100 deduction still leaves $17,983.53 of taxable income, which owes $1,910.02 at the 10%/12% rates." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It wrongly concluded that tax after the standard deduction was zero or negative. Wages of $34,083.53 minus the $16,100 deduction leave $17,983.53 taxable, which owes $1,910.02 with no nonrefundable credits." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules, with a combined deduction and exemption of $13,650 and a 15% second bracket. For 2026 the $16,100 standard deduction and the 10%/12% brackets apply, giving $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the §127 cap in taxable income, which pushed income into the 22% bracket. Educational assistance is excluded from AGI, so taxable income is $17,983.53 and tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, which raised income to $107,084. Educational assistance is excluded, so AGI is just the $34,083.53 of wages and the tax after the $16,100 deduction is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $19,000 reflects a 2025-level standard deduction of about $15,000 rather than the 2026 figure of $16,100. With $17,983.53 taxable and a $12,400 10% bracket, tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used 2025 parameters: a $15,000 standard deduction and an $11,925 10% bracket. The 2026 $16,100 deduction and $12,400 bracket give $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed post-TCJA-sunset law, with an $8,450 deduction, a $5,300 personal exemption and a 15% bracket. For 2026 the single standard deduction is $16,100 with no exemption under the 10%/12% brackets, so tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the right deduction and taxable income but ended the 10% bracket at $12,200 instead of the 2026 threshold of $12,400. That gave $1,914.08 instead of $1,240 + $670.02 = $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 of ESI premiums from the stated gross wages and applied pre-TCJA rules: an $8,527 deduction and a 15% bracket. The full $34,083.53 of wages minus the 2026 $16,100 deduction, taxed at 10%/12%, gives $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $1,910.02 reference." +us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and a $12,200 10% bracket instead of the 2026 figures of $16,100 and $12,400. That overstated taxable income at $18,334 and gave $1,956 instead of $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used the correct $16,100 deduction and $17,984 of taxable income, but its $1,915 matches a 10% bracket ending at $12,200 instead of the 2026 threshold of $12,400. The correct tax is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and a $12,150 10% bracket instead of the 2026 figures of $16,100 and $12,400. That gave $18,334 of taxable income and $1,957.08 of tax instead of $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It subtracted the EITC to bring tax to $0, but the EITC is refundable and is excluded from tax before refundable credits. The $16,100 deduction leaves $17,983.53 of taxable income, which owes $1,910.02." us,scenario_068,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly projected the temporary American Rescue Plan expansion of the childless EITC into 2026 and therefore used a phaseout endpoint near $33,000–$34,000. Under 2026 law, the temporary expansion is not in effect, and $34,084 exceeds the applicable childless EITC income limit, yielding zero EITC." us,scenario_068,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated wages of $34,084 and a one-person household as qualifying for EITC without applying the childless single-filer phaseout endpoint. At that income the childless EITC is fully phased out, and no other refundable credit applies." us,scenario_068,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract." @@ -5780,88 +5845,90 @@ us,scenario_068,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The us,scenario_068,payroll_tax,inkling,llm_error,state_local_rule,False,"The model got both FICA components right on the full $34,084 wage base and then added a 0.45% Maryland FAMLI employee contribution of $153.38. Maryland's FAMLI employee contributions are not in effect for 2026 and are not part of PolicyEngine's payroll_tax, which comprises only employee Social Security, Medicare, and Additional Medicare Tax." us,scenario_068,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive rule or computation was attempted. The required derivation is 6.2% plus 1.45% on the full $34,084 of wages, with no ESI offset, no educational assistance in the base, and no Maryland employee payroll contribution." us,scenario_068,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct base and rates but truncated each component to whole dollars ($2,113 Social Security and $494 Medicare) and then submitted $2,600 rather than the $2,607 those components sum to. The rule application was right; the reported total contradicts the model's own arithmetic by $7." -us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above $5,250 to income, making AGI $101,834, even though the full $73,000 stays out of Maryland AGI. It then used the old capped standard deduction of $2,700 instead of the flat $3,350 and halved the exemption because of that inflated AGI." -us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 of employer insurance premiums from wages, though they do not reduce Maryland AGI. Then, even with a $3,900 deduction leaving about $21,800 taxable, it declared the tax zero without applying the $3,200 exemption or the rate schedule. The correct taxable income is $27,533.53, which produces $1,255.34." -us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Claimed the taxable base was not positive. The $3,350 standard deduction and $3,200 exemption remove only $6,550 from $34,083.53 of AGI, leaving $27,533.53 taxable and $1,255.34 of tax." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Debated including $67,750 of educational assistance in income, worked out $4,554 and $1,336, and then submitted $1,029, which matches neither figure. Both of its paths used the old $2,700 standard deduction cap instead of Maryland's flat $3,350." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Used a standard deduction of 15% of AGI capped at $2,700 instead of the flat $3,350 Maryland adopted for 2025, which put taxable income at $28,184. It then added a made-up 'personal exemption credit adjustment' to move its own $1,286 up to $1,308." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the old standard deduction cap of about $2,700 instead of the flat $3,350, which overstated taxable income at $28,184. It also reported $1,301, while the rate schedule applied to its own $28,184 gives $1,286.24." -us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above $5,250 to Maryland income, making AGI $101,834, even though the whole $73,000 stays out of income. It also used a $2,500 standard deduction cap instead of the flat $3,350." -us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Used a capped standard deduction of about $2,550 instead of the flat $3,350, giving $28,334 of taxable income. It then reported $1,050, below the $1,293 that the 2%–4.75% schedule produces on its own taxable income, so the bracket arithmetic was also wrong." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put the full $73,000 of educational assistance into Maryland AGI, making it $107,084, though none of it belongs there. That inflated AGI triggered an exemption phase-out and the 5% bracket. It also used a $2,400 deduction cap and a $3,300 exemption instead of $3,350 and $3,200." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $93,445 AGI equals wages plus $67,750 of educational assistance minus $8,389 of insurance premiums. That is two mistakes: the educational assistance is not income, and the premiums do not reduce wages. Its $5,980 of combined deductions also falls short of the $6,550 from the flat $3,350 deduction and $3,200 exemption." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance to AGI, making it $101,834, used a $2,400 standard deduction instead of $3,350, and left out the $3,200 personal exemption entirely." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Got the AGI, the $3,200 exemption and the rate schedule right, but used a $2,800 standard deduction instead of Maryland's flat $3,350. That overstated taxable income by about $550, at $28,084 instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 of employer insurance premiums from wages for an AGI of $25,695, though they do not reduce Maryland AGI of $34,083.53. It also used the old standard deduction cap of $2,550 instead of the flat $3,350." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $1,133 implies about $24,960 of taxable income, so it took roughly $9,100 of deductions from $34,084 of wages instead of the correct $6,550 (the $3,350 standard deduction plus the $3,200 exemption), which leaves $27,533.53 taxable." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Subtracted the employer insurance premiums from wages, for an AGI of $25,695, and used a $2,750 standard deduction instead of $3,350. It then added an estimated $592 of county income tax, even though county tax is not part of state income tax before refundable credits." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $98,695 AGI equals wages plus the full $73,000 of educational assistance minus $8,389 of premiums. It should have excluded the educational assistance and left wages alone. It also used a $2,550 standard deduction and a $60 base on the first $3,000 instead of $90." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $997 implies about $22,100 of taxable income, roughly $5,400 below the correct $27,533.53 ($34,083.53 of AGI minus the $3,350 standard deduction and $3,200 exemption), so it took far too much off the income base." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Used a standard deduction 'maximum' of $2,550 from the old 15%-capped formula instead of the flat $3,350 that applies from 2025, which overstated taxable income at $28,334." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $4,786 matches about $101,800 of taxable income taxed partly in the 5% bracket, which only happens if the full $73,000 of educational assistance went into AGI. That assistance stays out of income, and correct taxable income is $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Used a $2,700 standard deduction from the old 15%-of-income cap instead of the flat $3,350, giving $28,184 of taxable income instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Used a $2,750 standard deduction and a $3,300 exemption instead of $3,350 and $3,200. It also misapplied the rates: $90 plus 4.75% of the excess over $3,000 on its own $28,034 is $1,279, not $1,453.28." -us,scenario_068,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used a $3,200 standard deduction instead of Maryland's flat $3,350, which overstated taxable income by $150 at $27,684. It then rounded $1,262.49 to $1,262." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction and exemption fully offset the wages. Together they total only $6,550 against $34,083.53 of AGI, leaving $27,533.53 taxable and $1,255.34 of tax." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Floored the tax at zero without ever computing taxable income. The $3,350 standard deduction and $3,200 exemption leave $27,533.53 taxable, and Maryland's 2%–4.75% schedule produces $1,255.34." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used an inflation-adjusted standard deduction of $2,900 instead of Maryland's flat $3,350, which overstated taxable income by $450 at $27,984." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above the $5,250 limit to Maryland AGI, making it $101,834, even though the full $73,000 stays out of income. AGI is just $34,083.53 of wages." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Used a projected standard deduction maximum of $2,850 from the repealed 15%-of-income formula instead of the flat $3,350, giving $28,034 of taxable income instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $101,334 taxable income comes from putting the full $73,000 of educational assistance into AGI ($107,084 minus $2,550 minus $3,200). That assistance stays out of income, and it also used $2,550 instead of the flat $3,350 standard deduction." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Used a $2,900 standard deduction instead of Maryland's flat $3,350, which overstated taxable income at $27,984 instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Used a $3,450 standard deduction instead of the flat $3,350. That understated taxable income by $100, at $27,434 instead of $27,533.53, and the tax by $4.75." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave a rough $1,400 without any deductions shown. That figure implies only about $3,500 was taken off wages, when the $3,350 standard deduction plus the $3,200 exemption remove $6,550 and give $1,255.34." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used a $2,350 'maximum' standard deduction from the old 15%-of-income formula instead of the flat $3,350, which overstated taxable income by $1,000 at $28,534." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Used a capped standard deduction of about $2,600 instead of Maryland's flat $3,350, giving $28,284 of taxable income instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Used the old 15%-of-AGI rule with a $2,850 cap for the standard deduction, though Maryland switched to a flat $3,350 for single filers starting in 2025. That gave $28,034 of taxable income instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 of employer insurance premiums from wages for an AGI of $25,695, though Maryland AGI is the full $34,083.53. It also used a $2,400 standard deduction instead of $3,350." -us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used a standard deduction of about $2,650 and an exemption of about $3,400 instead of the flat $3,350 and $3,200. That is $6,050 of deductions instead of $6,550, giving $28,034 of taxable income." -us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Treated the full $73,000 of educational assistance as taxable, making AGI $107,084, though it stays out of income. It also used a made-up 2%–6% rate schedule and a $2,700 standard deduction instead of the flat $3,350." -us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Used an inflation-adjusted standard deduction cap of $2,850 instead of the flat $3,350 that replaced the 15%-of-income formula in 2025, which overstated taxable income at $28,034." -us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Used a standard deduction of about $2,000 instead of $3,350 and left out the $3,200 personal exemption entirely, giving $32,084 of taxable income instead of $27,533.53." -us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Used a $2,350 'maximum' single standard deduction from the old 15%-capped formula instead of the flat $3,350, which overstated taxable income by $1,000 at $28,534." -us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Worked out $1,267.24 after deducting the $3,200 exemption and a $3,100 standard deduction (instead of $3,350). It then subtracted a made-up $240 'personal exemption credit' and a 'standard deduction credit equivalent', counting deductions Maryland allows only once a second time." -us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction and EITC bring taxable income to zero. EITC is a credit, not a deduction, and it is zero here, and the $3,350 standard deduction plus the $3,200 exemption leave $27,533.53 taxable." +us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance above the $5,250 Sec. 127 limit to AGI, getting $101,834. Maryland AGI is just the wages ($34,083.53). The model also used the old 15%-capped $2,700 standard deduction instead of the flat $3,350, and wrongly cut the exemption to $1,600." +us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premiums from wages to get AGI of $25,695, then claimed taxable income would be near zero. In fact $34,083.53 − $3,350 − $3,200 leaves $27,533.53 taxable, and Maryland's brackets give $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Said the Maryland tax on $34,084 of wages is not positive. The $3,350 standard deduction and $3,200 exemption leave $27,533.53 of taxable income, which produces $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own reasoning computed $4,554 (with the educational assistance included) and $1,336 (with it excluded), but it submitted $1,029, which matches neither. The model's exclusion branch also used the old $2,700 capped deduction instead of the flat $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Used the old 15%-of-AGI standard deduction capped at about $2,700 instead of Maryland's flat $3,350, which put taxable income at $28,184 and tax at $1,286. The model then added an unexplained 'personal exemption credit adjustment' to reach $1,308." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the old capped standard deduction of about $2,700 instead of the flat $3,350, giving taxable income of $28,184. The model then reported $1,301, more than the $1,286 that its own figures produce." +us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Put $67,750 of educational assistance into Maryland income, getting AGI of $101,834. Maryland AGI is just the wages ($34,083.53). The model also used a $2,500 capped standard deduction instead of the flat $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Used a $2,550 capped standard deduction instead of the flat $3,350, giving taxable income of $28,334. The model then got the bracket math wrong: tax on $28,334 is about $1,293, not the $1,050 it reported." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Added the full $73,000 of educational assistance to AGI, getting $107,084, which pushed income into the exemption phase-out and the 5% bracket. Maryland AGI is just the wages ($34,083.53), with the flat $3,350 deduction and full $3,200 exemption." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $93,445 AGI equals $34,084 + $67,750 of educational assistance − $8,389 of ESI premiums. Maryland AGI excludes the educational assistance entirely and does not subtract the premiums from the listed wages, leaving AGI of $34,083.53." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Added $67,750 of educational assistance to get AGI of $101,834, used a $2,400 capped deduction instead of $3,350, and left out the $3,200 personal exemption entirely." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The structure was right, but it used a $2,800 standard deduction instead of Maryland's flat $3,350 single deduction. That overstates taxable income by $550 and the tax by about $26." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premiums from wages, getting AGI of $25,695 instead of $34,083.53. The model also used a $2,550 capped deduction instead of the flat $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no computation. Its $1,133 implies taxable income of about $24,960, so about $9,100 of deductions came off $34,084. The correct reductions are the $3,350 standard deduction plus the $3,200 exemption, which leave $27,533.53 taxable and $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the ESI premiums to get AGI of $25,695 and used a $2,750 deduction instead of $3,350. The model then added about $592 of county income tax, which this state-tax output does not include." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $98,695 AGI equals wages plus the full $73,000 of educational assistance minus the $8,389 ESI premiums. Maryland AGI is just the wages ($34,083.53). The model also used a $2,550 deduction instead of $3,350, and the wrong $60 base for the first $3,000." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no computation. Its $997 implies taxable income of about $22,100, far below the correct $27,533.53 (wages of $34,083.53 less the $3,350 deduction and $3,200 exemption), which yields $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Used a $2,550 maximum standard deduction from the old 15%-of-AGI rule instead of the flat $3,350, which overstated taxable income ($28,334 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no computation. Its $4,786 implies taxable income of about $101,800, so it put the $73,000 of educational assistance into Maryland income. Maryland AGI is just the wages ($34,083.53)." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Used the old capped standard deduction of $2,700 instead of Maryland's flat $3,350, so taxable income came out at $28,184 instead of $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Used a $2,750 deduction and a $3,300 exemption instead of $3,350 and $3,200. The model also misapplied the brackets: its stated $28,034 of taxable income produces about $1,279, not $1,453.28." +us,scenario_068,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used a $3,200 standard deduction instead of Maryland's flat $3,350 single deduction. That overstates taxable income by $150 ($27,684 instead of $27,533.53) and the tax by about $7." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed deductions and exemptions fully offset the wages. Maryland allows only the $3,350 standard deduction and $3,200 exemption, which leaves $27,533.53 taxable and $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Floored the tax at zero without computing it. Wages of $34,083.53 less the $3,350 deduction and $3,200 exemption leave $27,533.53 taxable, which Maryland taxes at $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used an estimated $2,900 standard deduction instead of Maryland's flat $3,350 single amount, so taxable income came out at $27,984 instead of $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Used AGI of $101,834, which adds $67,750 of educational assistance to wages. Maryland AGI is just the wages ($34,083.53). The model also used the old maximum standard deduction instead of the flat $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Used a projected indexed cap of $2,850 under the old 15%-of-AGI rule instead of Maryland's flat $3,350 single deduction, which overstated taxable income by $500." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $101,334 of taxable income adds the full $73,000 of educational assistance to wages. Maryland AGI is just the wages ($34,083.53). The model also used a $2,550 deduction instead of $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Used a $2,900 standard deduction instead of Maryland's flat $3,350 single deduction, so taxable income came out at $27,984 instead of $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Used a $3,450 standard deduction instead of the flat $3,350 that Maryland published for 2025. That understates taxable income by $100 and the tax by about $4.72." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave a rounded guess of $1,400 with no deduction or exemption amounts. Applying the $3,350 deduction and $3,200 exemption to $34,083.53 gives $27,533.53 taxable and $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used a $2,350 maximum standard deduction from the old 15%-of-AGI rule instead of the flat $3,350, which overstated taxable income by $1,000." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Used a capped standard deduction of about $2,600 instead of Maryland's flat $3,350 single deduction, so taxable income came out at about $28,284 instead of $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Applied the old 15%-of-AGI rule with an indexed $2,850 cap. Maryland now uses a flat $3,350 single deduction, so taxable income was overstated by $500." +us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premiums from wages to get AGI of $25,695 instead of $34,083.53. The model also used a $2,400 deduction instead of the flat $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used about $2,650 for the deduction and $3,400 for the exemption instead of $3,350 and $3,200. That understates total reductions by $500 and puts taxable income at $28,034." +us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Treated the full $73,000 of educational assistance as taxable, getting AGI of $107,084, and applied a made-up 2%–6% rate schedule. Maryland AGI is just the wages ($34,083.53), taxed at 4.75% above $3,000." +us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Used an indexed $2,850 cap under the old 15% rule instead of Maryland's flat $3,350 single deduction, which overstated taxable income by $500." +us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Left out the $3,200 personal exemption entirely and used a $2,000 standard deduction instead of $3,350. That gives taxable income of $32,084 instead of $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its $28,084 of taxable income reflects about $6,000 of combined deduction and exemption. The flat $3,350 deduction plus the $3,200 exemption total $6,550, leaving $27,533.53 taxable." +us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Used a $2,350 maximum standard deduction under the old 15%-of-AGI rule instead of Maryland's flat $3,350, which overstated taxable income by $1,000." +us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Correctly subtracted a deduction (wrong amount, $3,100) and the exemption from income, then subtracted both again as made-up nonrefundable credits ($240 'personal exemption credit' plus a 'standard deduction credit'). Maryland has no such credits, so these double-count the deduction and exemption." +us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed taxable income is zero after the standard deduction and EITC. The EITC is a credit, not a deduction from taxable income. The $3,350 deduction and $3,200 exemption leave $27,533.53 taxable, and the tax on that is $1,255.34." us,scenario_068,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized that the childless federal EITC phases out to zero at this income but then invented a $245 residual Maryland refundable credit. Applying Maryland’s percentage to a federal EITC of zero yields $0, not a minimal positive amount." us,scenario_068,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated $34,084 as qualifying low income without applying the federal childless EITC phaseout ceiling, then assigned a Maryland EITC amount exceeding the credit pathway it cited. With no qualifying children, federal EITC is zero at this income, so the percentage-based Maryland refundable EITC and the total state refundable credits are both $0." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning correctly reaches taxable income of $20,170, tax of $2,172, and a 10% Saver's Credit of $192, for ≈$1,980. It then submitted 1697, which contradicts its own derivation. This is a transcription error, not a rule error." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 figure of $16,100. It also subtracted student loan interest after the standard deduction instead of including it in AGI. It never applied the 10% Saver's Credit ($192.40 on $1,924 of contributions), so it landed near the pre-credit tax at $2,180." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It misstated AGI as roughly $35,000 (correct: $36,270) and taxable income as about $18,000 (correct: $20,170). It also never subtracted the nonrefundable 10% Saver's Credit of $192.40, so its $2,010 matches neither the $2,172 pre-credit tax nor the $1,980 post-credit figure." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It correctly computed $2,172 of tax on $20,170 of taxable income, then raised it to $2,398 by an arbitrary 'bracket estimate adjustment.' It also left out the 10% Saver's Credit of $192.40 on $1,924 of retirement contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It computed $1,240 + $932 but stated the total as $2,549 instead of $2,172, an arithmetic error. It also said no nonrefundable credits apply, which misses the Saver's Credit: single-filer AGI of $36,270 qualifies at the 10% rate on $1,924 of contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It used a $16,900 standard deduction instead of the 2026 single amount of $16,100. It then replaced its own ≈$2,076 result with an unsupported $2,544. It also stated that no nonrefundable credits apply, omitting the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It used a projected $15,700 standard deduction instead of the OBBBA-set 2026 single amount of $16,100, which overstated taxable income by $400. It also concluded that no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40 on the $1,924 of 401(k) and IRA contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It correctly reached $2,172 of tax before credits. It then subtracted an unexplained ≈$843 to reach $1,329. At AGI $36,270 the Saver's Credit rate is 10%, so the credit is 10% of $1,924 = $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The Saver's Credit is correct at $192.40. However, it used a $15,600 standard deduction and a $12,300 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400. That overstated pre-credit tax at $2,234 versus $2,172." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $600 of health insurance premiums from wages even though they are not pre-tax payroll deductions here. It also used a $15,360 standard deduction instead of $16,100. Finally, it based the Saver's Credit only on traditional contributions ($1,554), excluding the Roth 401(k) and Roth IRA amounts that bring the eligible base to $1,924." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"AGI and the $192.40 Saver's Credit are correct. It used a $15,450 standard deduction instead of the 2026 single amount of $16,100, which inflated taxable income to $20,820 and pre-credit tax to $2,252.90." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $12,200 ceiling for the 10% bracket instead of the 2026 single threshold of $12,400. That gives pre-credit tax of $2,176.40 instead of $2,172.36, leaving it $4 high after the correct $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $7,600 standard deduction, a $5,150 personal exemption, and a 15% bracket. The OBBBA made TCJA rates permanent, so 2026 has a $16,100 single standard deduction, no personal exemption, and a 12% second bracket." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left the $276 student loan interest deduction out of AGI and used a $15,500 standard deduction instead of $16,100. Its $1,494 does not follow from any stated bracket computation. The correct path is $2,172.36 of tax less the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted a 'personal exemption' along with the standard deduction, which assumes the TCJA sunset. Under OBBBA, 2026 has no personal exemption, a $16,100 standard deduction, and 10%/12% brackets, so its tax is overstated by about $600." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a 'reverted' pre-TCJA standard deduction plus personal exemption and the 15% bracket. OBBBA made the TCJA structure permanent for 2026. It also wrongly subtracted health premiums when computing AGI ($35,970 instead of $36,270)." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It left out the $69 traditional IRA and $276 student loan interest deductions and used a $15,000 standard deduction and a $11,600 10% bracket instead of the 2026 values of $16,100 and $12,400. It also never applied the 10% Saver's Credit of $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The Saver's Credit is correct at $192.40. It used a $15,700 standard deduction and the 2025 $11,925 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400, which overstated pre-credit tax at $2,229.90." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a personal exemption under '2026 law,' which assumes the TCJA sunset. The OBBBA kept the $16,100 standard deduction, zero personal exemption, and 12% second bracket. Pre-credit tax should be $2,172.36, not about $2,780." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired and used an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The OBBBA made the TCJA rates permanent, so 2026 uses a $16,100 standard deduction with 10%/12% brackets, giving $2,172.36 before the Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"AGI and the $192.40 Saver's Credit are correct. It used an estimated $15,420 standard deduction and a $12,259 10%-bracket ceiling instead of the 2026 values of $16,100 and $12,400, which overstated pre-credit tax at $2,256.82." -us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly computed $2,172 of pre-credit tax but said no nonrefundable credits apply, considering only child-related credits and the EITC. The Saver's Credit applies at the 10% rate for single AGI of $36,270 on $1,924 of contributions, which reduces tax by $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed nonrefundable credits fully offset the tax, even though the household has no children or dependent care. The only applicable credit is the $192.40 Saver's Credit, which leaves $1,979.96 on $2,172.36 of tax." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed nonrefundable credits reduce tax to zero without naming any qualifying credit. On $20,170 of taxable income the tax is $2,172.36, and the only available nonrefundable credit is the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a 'post-TCJA-sunset' standard deduction plus personal exemption and the 15% bracket, but the OBBBA made the TCJA rates permanent for 2026. It also said no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It correctly reached $2,172.40 of pre-credit tax but asserted that no nonrefundable credits are available. It missed the Saver's Credit: single AGI of $36,270 falls in the 10% tier, so the credit on $1,924 of 401(k) and IRA contributions is $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It computed the no-credit tax of $2,172.40 and stopped there. It never applied the nonrefundable Saver's Credit of 10% on $1,924 of traditional and Roth 401(k)/IRA contributions ($192.40), which applies at single AGI of $36,270." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It reached the correct $2,172.40 of pre-credit tax but said no nonrefundable credits were used. That leaves out the $192.40 Saver's Credit (10% tier at AGI $36,270, on $1,924 of contributions)." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave $1,623 with no computation. The correct taxable income of $20,170 yields $2,172.36 of tax, and the $192.40 Saver's Credit leaves $1,979.96. Its figure understates taxable income or overstates credits by about $357." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law sunset rules' with an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The OBBBA made the TCJA structure permanent, so the 2026 standard deduction is $16,100 with no exemption and a 12% second bracket. Its Saver's Credit of $192 is correct." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used post-TCJA-sunset rules (personal exemption, 10%/15% brackets) to reach $22,670 of taxable income. Under OBBBA the 2026 single standard deduction is $16,100 with no exemption, giving $20,170 taxed at 10%/12%. Its $192 Saver's Credit is correct." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $2,176 of ordinary tax reflects a $12,200 10%-bracket ceiling instead of the 2026 single threshold of $12,400, which gives $2,172.36. That leaves its answer $4 high after the correct $192 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It wrongly subtracted health insurance premiums from wages (AGI $35,970 instead of $36,270) and used a $15,400 standard deduction instead of $16,100. It also limited the Saver's Credit to traditional contributions ($155), omitting the Roth 401(k) and Roth IRA amounts that make the credit $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"It filed as head of household even though the head has no qualifying person, so the status is single. It therefore used a $21,900 standard deduction and wider HoH brackets instead of the single $16,100 and $12,400. It also computed the Saver's Credit only on traditional contributions ($1,554) instead of all $1,924." -us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It estimated about $2,108 of tax and a $159 Saver's Credit, then submitted 0 based on unspecified 'other nonrefundable credits' that this childless single filer does not have. The only credit is the $192.40 Saver's Credit, which leaves $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% Saver's Credit rate, claiming AGI was below the lowest single threshold. Single AGI of $36,270 is far above the 50%-tier ceiling and falls in the 10% tier, so the credit is $192.40, not $962." -us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It left the $276 student loan interest deduction out of AGI and used a $15,750 standard deduction instead of $16,100. It applied a 50% Saver's Credit rate on traditional contributions only, when AGI of $36,270 qualifies for just 10% of the full $1,924. It then submitted $1,213.50, which contradicts its own $1,475.52." -us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It claimed the Credit for Other Dependents eliminates the tax, but the household has no dependents, so that credit is $0. It also overstated pre-credit tax at $3,537 instead of $2,172.36. The only applicable credit is the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning gets the full chain right: taxable income $20,170, tax $2,172, a 10% Saver's Credit of $192, and a result of about $1,980. It then submitted 1,697, which none of its stated steps support, so this is a transcription/output error." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It used the stale $14,600 standard deduction instead of 2026's $16,100, and its tax arithmetic was muddled. It never applied the nonrefundable Saver's Credit (10% × $1,924 = $192.40) that brings the $2,172 tax down to $1,980." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It put taxable income near $18,000; the correct figure is $20,170 (AGI $36,270 minus $16,100). It also never subtracted the $192.40 Saver's Credit that applies at the 10% rate for AGI of $36,270." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It correctly reached $2,172 of tax on $20,170 of taxable income, then inflated it without basis to $2,398. It also missed the 10% Saver's Credit on $1,924 of qualifying 401(k)/IRA contributions ($192.40)." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It added $1,240 + $932 and reported $2,549 instead of $2,172, an arithmetic error. It also said no nonrefundable credits apply, missing the $192.40 Saver's Credit (10% of $1,924 at AGI $36,270)." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $16,900 standard deduction instead of 2026's $16,100, and its submitted $2,544 does not match its own ~$2,076 computation. It also left out the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It used a projected $15,700 standard deduction instead of 2026's $16,100, which overstated taxable income by $400. It also said no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40 on $1,924 of 401(k)/IRA contributions." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It correctly computed $2,172 of tax, then subtracted about $843 as a Saver's Credit. At AGI $36,270 a single filer gets only the 10% rate, so the credit is $192.40 (10% of $1,924) and the result is $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used a $15,600 standard deduction and a $12,300 top for the 10% bracket instead of 2026's $16,100 and $12,400. That inflated tax before credits to $2,234.40 instead of $2,172.36." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $600 of health insurance premiums from wages, though they are not a pre-tax exclusion here, and it used a $15,360 standard deduction instead of $16,100. It also limited the Saver's Credit to the $1,554 of traditional contributions; the qualifying base is $1,924 including the Roth 401(k) and Roth IRA." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,450 standard deduction instead of 2026's $16,100, which put taxable income at $20,820 instead of $20,170 and tax at $2,252.90 instead of $2,172.36. It then applied the correct $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $12,200 top for the 10% bracket instead of 2026's $12,400. That taxed $200 at 12% instead of 10% and gave $2,176.40 of tax instead of $2,172.36, before the correct $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and used a $7,600 standard deduction, a $5,150 personal exemption and a 15% bracket. The OBBBA made TCJA rates permanent: 2026 has a $16,100 standard deduction, no personal exemption, and a 12% bracket above $12,400." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left the $276 student loan interest deduction out of AGI and used a $15,500 standard deduction instead of $16,100. Its $1,494 does not follow from applying the 10%/12% brackets to its own taxable income. The correct path is $20,170 taxable, $2,172.36 tax, less the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied a personal exemption and reverted pre-TCJA rules for 2026. Under the OBBBA, 2026 has no personal exemption, a $16,100 single standard deduction, and 10%/12% brackets, giving $2,172.36 of tax before the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the 2026 TCJA sunset (reverted standard deduction plus personal exemption, 15% bracket) and also subtracted health insurance premiums from AGI. The correct 2026 law uses a $16,100 standard deduction and 10%/12% brackets on $20,170 of taxable income." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It left the $69 traditional IRA and $276 student loan interest deductions out of AGI and used a stale $15,000 standard deduction with the 2024 $11,600 bracket. It also never applied the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $15,700 standard deduction and the 2025 $11,925 top for the 10% bracket instead of 2026's $16,100 and $12,400, which produced $2,229.90 of tax instead of $2,172.36. Its $192.40 Saver's Credit was correct." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a personal exemption under an assumed TCJA sunset. Under 2026 law there is no personal exemption, the standard deduction is $16,100 and the brackets are 10%/12%, which gives $2,172.36 before the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket on the assumption that TCJA expired. The OBBBA made TCJA permanent, so 2026 uses a $16,100 standard deduction and 12% above $12,400, putting taxable income at $20,170, not $22,670." +us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $15,420 standard deduction and a $12,259 top for the 10% bracket instead of 2026's published $16,100 and $12,400, which overstated tax at $2,256.82. Its $192.40 Saver's Credit was correct." +us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly computed $2,172 of tax but said no nonrefundable credits apply. It missed the Saver's Credit: 10% of $1,924 in 401(k)/IRA contributions (Roth included) at AGI $36,270, which is $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It claimed nonrefundable credits wipe out the whole liability, yet the household has no dependents, so no CDCC or CTC applies. The only nonrefundable credit is the $192.40 Saver's Credit against $2,172.36 of tax on $20,170 of taxable income." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed nonrefundable credits reduce tax to zero. With no dependents, the only nonrefundable credit is the 10% Saver's Credit ($192.40), which leaves $1,979.96 of the $2,172.36 tax." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a projected post-TCJA-sunset standard deduction, a personal exemption and a 15% bracket. 2026 law keeps the $16,100 standard deduction and 10%/12% brackets. It also said no nonrefundable credits apply, missing the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It correctly computed $2,172.40 of tax but said no nonrefundable credits are available. It omitted the Saver's Credit of 10% × $1,924 = $192.40, which applies because AGI of $36,270 is within the single 10%-rate tier." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It stopped at the $2,172.40 tax figure and applied no credits. It missed the nonrefundable Saver's Credit of $192.40 on $1,924 of traditional and Roth 401(k)/IRA contributions at the 10% AGI tier." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It correctly computed $2,172.40 of tax but said no nonrefundable credits are used. AGI of $36,270 qualifies for the 10% Saver's Credit on $1,924 of contributions, which is $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no computation. The correct path is $20,170 of taxable income, $2,172.36 of tax and a $192.40 Saver's Credit, giving $1,979.96. Its $1,623 implies about $549 of credits or deductions that do not exist." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law sunset' rules ($8,300 standard deduction, $5,300 personal exemption, 15% bracket). The OBBBA made the TCJA structure permanent, so 2026 uses a $16,100 standard deduction and 10%/12% brackets. Its $192 Saver's Credit was correct." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed tax on $22,670 of taxable income using the post-TCJA-sunset standard deduction, a personal exemption and the 15% bracket. 2026 law gives $20,170 of taxable income under the $16,100 standard deduction and 10%/12% brackets. Its $192 Saver's Credit was correct." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $2,176 ordinary tax reflects a $12,200 top for the 10% bracket instead of 2026's $12,400; the correct tax is $2,172.36. Its standard deduction and $192 Saver's Credit were right." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"It subtracted health insurance premiums from AGI, used a $15,400 standard deduction instead of $16,100, and computed the Saver's Credit on traditional contributions only ($155). The qualifying base is $1,924, which includes the $262 Roth 401(k) and $108 Roth IRA, so the credit is $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"It filed as head of household with a $21,900 standard deduction and HoH brackets. A lone adult with no qualifying person must file single, with a $16,100 deduction and the 10% bracket ending at $12,400. It also left Roth contributions out of the Saver's Credit base." +us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It computed about $2,108 of tax and a Saver's Credit, then claimed unnamed 'other nonrefundable credits' reduce the liability to zero. No other nonrefundable credits apply, so the result is $2,172.36 minus $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"It correctly computed $2,172.40 of tax on $20,170 of taxable income but said no nonrefundable credits apply. It missed the 10% Saver's Credit ($192.40) on $1,924 of retirement contributions." +us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% Saver's Credit rate, claiming AGI was below the lowest single threshold. AGI of $36,270 is above both the 50% and 20% tier limits for single filers, so only the 10% rate applies, for a credit of $192.40, not $962." +us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It applied a 50% Saver's Credit rate to traditional contributions only ($777). The correct credit is 10% of $1,924, including the Roth contributions, at AGI $36,270. It also left the student loan interest deduction out of AGI, used a $15,750 standard deduction, and submitted 1,213.5 against its own $1,475.52." +us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied the Credit for Other Dependents to eliminate the tax, but this single filer has no dependents. It also overstated tax before credits at $3,537; the correct figure is $2,172.36, reduced only by the $192.40 Saver's Credit." us,scenario_070,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied the Additional Child Tax Credit earned-income formula despite the household having no qualifying child, so there was no Child Tax Credit balance available to refund. Its separate EITC rationale also misstates the rule: $100 of investment income does not exceed the limit; the childless EITC is zero because $38,000 of earnings exceeds its phaseout ceiling." us,scenario_070,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model called eligibility borderline without calculating the stated household’s MAGI-to-FPL ratio. At 2.27 times FPL, the head exceeds Illinois’s 138% FPL ACA expansion limit, and age 57, low assets, and employer-sponsored insurance do not establish another Medicaid category." us,scenario_070,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $300 health insurance premium as a pre-tax FICA exclusion even though no pre-tax treatment was specified. FICA applies to the full $38,000, yielding $2,356 of Social Security tax and $551 of Medicare tax." @@ -5869,42 +5936,43 @@ us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"T us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model conflated regular Medicare tax with the Additional Medicare Tax and set all Medicare tax to zero because wages were below the additional-tax threshold. The ordinary 1.45% Medicare tax applies from the first dollar of covered wages, adding $551 to the correctly computed $2,356 Social Security tax." us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $38,000 to $37,700 by assigning unlisted pre-tax treatment to the $300 health premium. Applying 6.2% Social Security and 1.45% Medicare rates to the full wages produces $2,907." us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with $2,970 under an invalid claim of rounding. Rounding $2,907 does not produce $2,970, and the requested annual amount is the exact derived total." -us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Used the 2025 Illinois personal exemption of $2,850 instead of the 2026 amount of $2,925. That leaves taxable income at $33,420 instead of $33,344.67, so the tax comes to $1,654.29 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Left the $1,485 traditional 401(k) deferral in the base, which Illinois picks up from federal AGI. Treated the exemption as a $120 credit instead of a $2,925 deduction from base income. Its own figures ($1,869 − $120) don't add up to the $1,154 it submitted; the correct result is 4.95% × $33,344.67 = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Applied a nonexistent Illinois 'retirement subtraction' that cut the base to about $20,000. Illinois base income is federal AGI ($36,270) minus only the $2,925 exemption, which gives $33,344.67 and $1,650.56 of tax." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used the outdated $2,850 exemption instead of the 2026 amount of $2,925. It then raised its own $1,654.29 result to $1,659.83 with no basis, when the correct figure is $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. It then dropped its $1,658 result and substituted an unsupported taxable income of $34,870. The correct taxable income is $36,270 − $2,925 = $33,344.67, giving $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Used the $2,850 exemption instead of the 2026 amount of $2,925, then raised its own ~$1,655 figure to $1,699 for 'rounding' with no basis. The correct liability is 4.95% × $33,344.67 = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Added back the IRA deduction and student loan interest, but Illinois has no such add-backs because base income is federal AGI. Subtracted the $300 health premium as a pre-tax wage reduction even though the facts don't say it is pre-tax. Used the 2023 exemption of $2,425 instead of $2,925, producing $33,890 of taxable income instead of $33,344.67." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Left out the $276 student loan interest deduction on the claim that Illinois disallows it, but the deduction is already built into federal AGI, which is where Illinois starts. Used the $2,850 exemption instead of $2,925, then added an unsupported 'phase-in' adjustment to reach $1,690." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Subtracted the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $33,420 of taxable income and $1,654 of tax instead of $33,344.67 and $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Invented an Illinois standard deduction of $2,450 and stacked it on a $2,450 exemption. Illinois allows only the personal exemption, which is $2,925 for 2026, so taxable income is $33,344.67, not $31,370." -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Understated AGI by $600 ($35,670 instead of about $36,270), which matches subtracting both $300 health premium entries, neither of which reduces wages. Also used a $2,842 exemption instead of the 2026 amount of $2,925." -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $1,654.29 instead of 4.95% × $33,344.67 = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the 2024 exemption of $2,775 instead of $2,925. It also invented a $135 nonrefundable renter's credit, but Illinois's property tax credit applies only to property tax paid on a principal residence and is not available to renters on rent." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of $2,925. Its $1,716 answer implies about $34,667 of taxable income, which means it started from an inflated base of roughly $37,440 instead of the $36,270 federal AGI." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Subtracted a rough $2,500 exemption instead of the 2026 amount of $2,925. That leaves taxable income at about $33,770 instead of $33,344.67 and tax at $1,671 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Took $300 off AGI ($35,970 instead of about $36,270) by treating a health insurance premium as a pre-tax wage reduction, which the facts don't support. Also used the 2024 exemption of $2,775 instead of $2,925." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $1,683 implies $34,000 of taxable income, which means it took only about $2,270 off the $36,270 AGI. The 2026 exemption is $2,925, giving $33,344.67 of taxable income and $1,650.56 of tax." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $33,420 of net income and $1,654.29 of tax instead of $33,344.67 and $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That overstates taxable income by $150 and gives $1,658 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That leaves taxable income at $33,495 instead of $33,344.67 and tax at $1,658 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Left the $276 student loan interest deduction out of federal AGI ($36,546 instead of $36,270) and used the 2023 exemption of $2,425 instead of $2,925. It also submitted $1,626, which doesn't match its own $1,689 calculation." -us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Subtracted the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $1,654 instead of 4.95% × $33,344.67 = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the exemption and retirement deductions wipe out Illinois tax. Federal AGI of $36,270 minus the single $2,925 exemption still leaves $33,344.67 taxable at 4.95%, so the liability is $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Concluded the retirement and student loan adjustments bring Illinois liability to zero. After those adjustments, AGI is still $36,270 and only a $2,925 exemption comes off, leaving $33,344.67 taxed at 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Left the $276 student loan interest deduction out of federal AGI ($36,546 instead of $36,270). It also used an exemption of $1,824 instead of the 2026 amount of $2,925, which overstates taxable income at $34,722." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. That gives $1,654.29 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $1,777 implies about $35,900 of taxable income, so it took only about $370 off the $36,270 AGI instead of the full $2,925 exemption. The correct taxable income is $33,344.67, giving $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That gives $1,658 instead of 4.95% × $33,344.67 = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Understated AGI as $35,970 by subtracting a $300 health premium as though it were a pre-tax wage reduction, and used the 2023 exemption of $2,425 instead of $2,925. The two errors roughly cancel, but the result is still $1,660 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Used the 2024 exemption of about $2,775 instead of the 2026 amount of $2,925. That leaves taxable income at $33,495 instead of $33,344.67 and tax at $1,658 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Stated that Illinois has no personal exemption and taxed the full $36,270 AGI. Illinois allows a $2,925 personal exemption in 2026, which brings taxable income down to $33,344.67 and tax to $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That gives $33,495 of net income and $1,658 of tax instead of $33,344.67 and $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Applied a nonexistent Illinois standard deduction that cut taxable income to about $20,696, then made an unsupported upward adjustment to $1,106. Illinois taxes federal AGI less only the $2,925 exemption, which is $33,344.67 here." -us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Estimated the 2026 exemption at $2,900 instead of the actual $2,925. That leaves taxable income at $33,370 instead of $33,344.67 and tax at $1,651.82 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Counted the exemption twice: once as a deduction from income and again as a $2,800 dollar-for-dollar nonrefundable credit, which zeroed out the tax. Illinois has only the exemption deduction ($2,925 in 2026), so the liability is $1,650.56. It also left the $276 student loan interest out of AGI." -us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its $1,658.66 implies about $33,508 of taxable income, which matches subtracting the 2024 exemption of about $2,775 instead of the 2026 amount of $2,925. The correct taxable income is $33,344.67, giving $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model correctly started from federal AGI of $36,270 but subtracted the 2025 Illinois exemption of $2,850 instead of the 2026 indexed exemption of $2,925. That overstated taxable income by $75 and tax by about $3.70." +us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left the $1,485 traditional 401(k) deferral in the base, which it called 'federal taxable income' of $37,755, and applied the outdated $2,425 exemption as a $120 credit instead of a $2,925 income subtraction. Its own arithmetic also breaks down: $1,869 minus $120 is $1,749, not $1,154. The correct calculation is ($36,270 − $2,925) × 4.95% ≈ $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model invented a 'retirement subtraction' that cut the Illinois base to about $20,000. Illinois has no such subtraction for wage earners' retirement contributions beyond what is already excluded from federal AGI. The correct base is AGI of $36,270 minus the $2,925 exemption, or $33,345." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 amount of $2,925 to get $1,654.29. It then raised that to $1,659.83 with no basis, which moved further from the correct ($36,270 − $2,925) × 4.95%." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed AGI of $36,270 correctly but used the outdated $2,775 exemption, which gave $1,658. It then switched to an unexplained taxable income of $34,870 and answered $1,726, instead of applying the 2026 exemption of $2,925 for a base of $33,345." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 amount of $2,925, which gave about $1,655. It then added an arbitrary 'rounding' adjustment to reach $1,699." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $300 health premium from wages as if it were a pre-tax payroll deduction, and it wrongly added back the IRA deduction and student loan interest. Illinois base income follows federal AGI, and those items stay deducted. It also used the outdated $2,425 exemption instead of $2,925, which overstated taxable income by about $545." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model left the $276 student loan interest deduction out of AGI, but Illinois starts from federal AGI, which already reflects that deduction. It also used the 2025 exemption of $2,850 instead of $2,925, then applied an invented 'phase-in' adjustment to reach $1,690." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the 2025 Illinois exemption of $2,850 instead of the 2026 indexed amount of $2,925. That produced $33,420 of taxable income and $1,654 of tax, instead of $33,345 and $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted a $2,450 'Illinois standard deduction' that does not exist, plus an outdated $2,450 exemption. Illinois allows only the personal exemption, which is $2,925 in 2026, against federal AGI of $36,270." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model lowered AGI to $35,670 by also subtracting the $600 in listed health premiums, which are not AGI adjustments here. It also used a $2,842 exemption instead of the 2026 amount of $2,925." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied the 2025 exemption of $2,850 to AGI of $36,270 instead of the 2026 indexed exemption of $2,925. That overstated tax by about $3.70." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used the outdated $2,775 exemption and subtracted a $135 'renter's credit' that does not exist. The Illinois 5% property tax credit applies only to real property taxes paid on a residence, not to a share of rent. No nonrefundable credit applies, and the exemption should have been $2,925." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $1,716 implies about $34,667 of taxable income, roughly $1,320 above the correct $33,345. That is consistent with using the outdated $2,775 exemption on a base that did not fully remove the 401(k), IRA, and student loan adjustments from the $36,270 AGI." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model subtracted a 'roughly $2,500' exemption from AGI of $36,270 instead of the 2026 Illinois exemption of $2,925. That overstated taxable income by about $425." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model lowered AGI to $35,970 by treating the $300 health premium as an above-the-line deduction, which it is not here. It also used the outdated $2,775 exemption instead of $2,925." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model's $1,683 implies $34,000 of taxable income, about $655 above the correct $36,270 − $2,925 = $33,345. That is consistent with using an outdated, smaller exemption and not fully applying the AGI adjustments." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925, which overstated taxable income by $75." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That overstated taxable income by $150 and tax by about $7.40." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2024 exemption of $2,775 instead of the 2026 indexed exemption of $2,925, which gave $1,658 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model left the $276 student loan interest deduction out of AGI, which it put at $36,546, and used the outdated $2,425 exemption. That gave $1,689, and the model then submitted an inconsistent $1,626. The correct calculation is ($36,270 − $2,925) × 4.95%." +us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 amount of $2,925 and rounded the result to $1,654." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that the exemption and retirement deductions reduce Illinois tax to zero. In fact, AGI of $36,270 far exceeds the single $2,925 exemption, which leaves $33,345 taxed at 4.95%." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model returned zero without computing anything, but Illinois taxes $36,270 of AGI minus the $2,925 exemption at a flat 4.95%. No nonrefundable credit offsets that tax." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model left the $276 student loan interest deduction out of AGI, which it put at $36,546, and used an invented $1,824 exemption instead of the 2026 Illinois exemption of $2,925. Together these overstated taxable income by about $1,380." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925, which gave $1,654.29 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $1,777 implies about $35,900 of taxable income, which roughly equals $38,100 of gross income minus an exemption of about $2,200. That means it did not subtract the 401(k), IRA, and student loan adjustments to reach the $36,270 AGI, and it understated the $2,925 exemption." +us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model subtracted the 2024 exemption of $2,775 from AGI of $36,270 instead of the 2026 exemption of $2,925, which overstated tax by about $7.40." +us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model lowered AGI to $35,970 by wrongly subtracting the $300 health premium, and it used the outdated $2,425 exemption instead of $2,925. The net effect overstated taxable income by about $200." +us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used the 2024 exemption of $2,775 instead of the 2026 indexed exemption of $2,925, which gave $1,658 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model stated that Illinois has no personal exemption and taxed the full $36,270 AGI. Illinois allows a personal exemption of $2,925 for 2026, which reduces taxable income to $33,345." +us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly used AGI of $36,270 and found no credits, but subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925." +us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied a 'standard deduction' that brought taxable income to about $20,696, but Illinois has no standard deduction. The correct base is federal AGI of $36,270 minus only the $2,925 personal exemption." +us,scenario_070,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model used an outdated $2,450 exemption instead of the 2026 indexed amount of $2,925. That overstated taxable income by $475 and tax by about $23.50." +us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model estimated the 2026 exemption at about $2,900 instead of the actual $2,925. That overstated taxable income by about $25 and gave $1,651.82." +us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted the exemption from income and then also subtracted $2,800 as a dollar-for-dollar 'personal exemption credit' against the $1,670 of tax, which drove the result to zero. The Illinois exemption is only an income subtraction of $2,925 in 2026, and the model also omitted the student loan interest deduction from AGI." +us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's $1,658.66 implies about $33,508 of taxable income, which is consistent with subtracting an outdated exemption of roughly $2,775 from AGI. The 2026 indexed exemption is $2,925." us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly found that the childless federal EITC was fully phased out but then invented a $102 Illinois EITC despite Illinois calculating its credit as a percentage of the federal amount. The age expansion changes categorical eligibility only; it does not override the federal income phaseout, so 20% of a zero federal EITC yields $0." us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model declared $36,542 of income below New York’s aged Medicaid limits and simultaneously treated $62,291 as below a resource limit it described as $15,900, reversing both comparisons. It therefore invented an aged Medicaid pathway even though the person’s engine category is NONE." us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invoked unspecified senior disregards to reduce $36,542 below an aged Medicaid limit of roughly $20,000 without calculating any applicable disregard. No aged pathway is established, and the engine assigns medicaid_category = NONE." @@ -5948,69 +6016,71 @@ us,scenario_073,head_medicaid_eligible,gpt-6.1-sol,llm_error,taxable_income_or_d us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model said only taxable income counts for Medicaid MAGI and dropped the $22,646 of SSDI. Medicaid MAGI is AGI plus untaxed Social Security, tax-exempt interest and excluded foreign income, so the full SSDI counts. Total MAGI is $25,754, or 1.60 times FPL, above the 138% expansion limit." us,scenario_073,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model applied the tax-law rule that Social Security counts only to the extent it is taxable. Medicaid MAGI instead adds all untaxed Social Security benefits back. MAGI is therefore $25,754, or 1.60 times FPL, not $3,108, which is above the 138% FPL expansion limit it cited." us,scenario_073,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the lack of wages as settling eligibility and never ran an income test. The $22,646 of SSDI and $3,108 pension give a Medicaid MAGI of 1.60 times FPL, above Michigan's 138% expansion limit. The head receives no SSI and has no dependent children, so no other category applies." -us,scenario_073,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"It named the 24-month SSDI waiting period correctly, then took the full-year, no-change assumption as proof the period was met. That assumption covers only the tax year, which is 12 months at most. The prompt's default makes the unlisted months of prior SSDI receipt 0, so the 24-month test fails and the head is not eligible." -us,scenario_073,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It said receiving Social Security disability income by itself makes the head Medicare eligible and never applied the 24-month waiting period. The months of SSDI receipt are unlisted and count as 0, so the disability route is not met and the 57-year-old head is not eligible." -us,scenario_073,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It stated the 24-month SSDI rule, then assumed disability status was already established. That assumption adds a fact the household does not list. Under the prompt's rule the months of SSDI receipt are 0, so the waiting period is not met and the head is not eligible." -us,scenario_073,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It noted that Medicare for people under 65 on SSDI starts only after 24 months, then treated the head as eligible anyway. Nothing in the household shows 24 months of receipt. The months of SSDI receipt default to 0, so the disability route fails and the answer should have been 0." -us,scenario_073,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"It treated current SSDI receipt as granting Medicare regardless of age and left out the 24-month waiting period. With the months of receipt unlisted and set to 0, the head is 57 and qualifies by neither age nor disability." -us,scenario_073,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It read the full-year, ongoing SSDI as meaning the 24-month waiting period was met. A benefit that is steady within the tax year shows at most 12 months of receipt. The prompt sets unlisted months of receipt to 0, so the head fails the 24-month test and is not eligible." -us,scenario_073,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It treated the head as having finished the SSDI waiting period, but no household fact says so. The prompt's default makes the months of SSDI receipt 0, so the 24-month requirement is not met and the 57-year-old head is not Medicare eligible." -us,scenario_073,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It counted being an SSDI recipient as enough for Medicare and skipped the 24-month waiting period. The months of receipt are unlisted and count as 0, so the head meets neither the age-65 test nor the disability test." -us,scenario_073,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It treated receiving SSDI as automatic Medicare eligibility and ignored the 24-month waiting period. With the months of receipt at their default of 0 and the head aged 57, no route to Medicare applies." -us,scenario_073,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It linked current SSDI receipt directly to Medicare without the 24-month waiting period. The household gives no duration of SSDI receipt, which defaults to 0, so the disability route is not met and the head is not eligible." -us,scenario_073,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It correctly said a 57-year-old needs a qualifying disability status, then treated the SSDI income as evidence that status was met. Disability-based Medicare requires 24 months of benefits, and the unlisted months of receipt count as 0, so the head is not eligible." -us,scenario_073,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"It took a full year of SSDI income as meeting the disability route. That route requires 24 months of benefits, and one year of income shows at most 12. The prompt's default of 0 for unlisted months of receipt means the head is not eligible." -us,scenario_073,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It said SSDI receipt establishes disability-based Medicare for someone under 65 and skipped the 24-month waiting period. The unlisted months of SSDI receipt count as 0, so the waiting period is not met and the head is not eligible." -us,scenario_073,head_medicare_eligible,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It treated receiving SSDI as automatically meeting the disability route to Medicare without the 24-month waiting period. The months of receipt are unlisted and set to 0, so the 57-year-old head qualifies by neither age nor disability." -us,scenario_073,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"It applied the disability route based only on current SSDI receipt and left out the 24-month waiting period. With months of receipt at the prompt's default of 0, the route fails and the head is not eligible." -us,scenario_073,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"It counted SSDI receipt alone as enough for disability-based Medicare, skipping the rule that benefits must be received for 24 months first. Unlisted months of receipt count as 0, so the head is not Medicare eligible." -us,scenario_073,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"It assumed the 24-month SSDI waiting period was met for an ongoing disability, but the months of receipt are unlisted and count as 0. It also misread the input name 'health insurance premiums excluding Medicare Part B' as evidence of Medicare enrollment. That label only names the premium variable and says nothing about Part B coverage." -us,scenario_073,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"It said SSDI receipt gives Medicare eligibility under PolicyEngine rules and never applied the 24-month waiting period. With the months of receipt at their default of 0 and the head aged 57, the head is not eligible." +us,scenario_073,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly noted that the 24-month SSDI waiting period applies, but then treated it as met because of the full-year constant-status assumption. That assumption covers only 12 months of receipt. The months-of-SSDI-receipt input is unlisted and therefore 0, so the 24-month disability pathway fails and a 57-year-old is not eligible." +us,scenario_073,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated any Social Security disability income as automatic Medicare eligibility. Medicare's disability pathway requires 24 months of SSDI receipt, and the months-of-receipt input is unlisted and therefore 0. With the head at 57 and no ESRD listed, they are not eligible." +us,scenario_073,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model named the 24-month SSDI requirement but simply assumed the disability status was established. The prompt says to treat unlisted numeric inputs as 0, so months of SSDI receipt are 0 and the head does not meet the 24-month test. At 57, they also miss the age-65 test." +us,scenario_073,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI rule but treated the head as having met it with no listed duration of receipt. Months of SSDI receipt are an unlisted input and therefore 0, so the disability pathway fails. At 57, the age pathway fails too." +us,scenario_073,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model said SSDI receipt alone confers Medicare eligibility under 65 and left out the 24-month waiting period. Months of SSDI receipt are unlisted and therefore 0, so the 57-year-old head is not eligible." +us,scenario_073,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model treated a full-year, ongoing SSDI benefit as proof that the 24-month waiting period was met. A constant status within one tax year shows at most 12 months of receipt. The months-of-receipt input is unlisted and therefore 0, so the head misses the 24-month threshold and is not eligible." +us,scenario_073,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model asserted that the head had completed the SSDI waiting period, but no household fact supports that. Months of SSDI receipt are unlisted and therefore 0, below the 24-month requirement. At 57, the head is not Medicare eligible." +us,scenario_073,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model treated being an SSDI recipient as enough for Medicare. People under 65 also need 24 months of SSDI receipt, and that input is unlisted and therefore 0. The 57-year-old head fails both the age and disability tests." +us,scenario_073,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model treated SSDI income as automatically qualifying the head for Medicare and never applied the 24-month SSDI receipt requirement. Months of receipt are unlisted and therefore 0, so the 57-year-old head is not eligible." +us,scenario_073,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare eligibility and skipped the 24-month waiting period. Months of SSDI receipt default to 0 because they are unlisted, so the 57-year-old head fails the disability pathway and is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model said under-65 eligibility requires meeting Medicare's disability rules, then accepted the SSDI income line as evidence that those rules were met. The rule that matters is 24 months of SSDI receipt. That input is unlisted and therefore 0, so the head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model concluded that full-year SSDI income alone met the disability pathway. That pathway requires 24 months of SSDI receipt, and one full year shows only 12; the listed months of receipt are 0. The 57-year-old head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model said SSDI income establishes disability-based Medicare eligibility without checking the 24-month receipt requirement. Months of SSDI receipt are unlisted and therefore 0, so the head, at 57 and without ESRD, is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model treated receipt of SSDI income as enough for the Medicare disability pathway and ignored the 24-month waiting period. Months of receipt are an unlisted input and therefore 0, so the 57-year-old head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model applied the disability-benefit pathway based only on SSDI income being present. That pathway requires at least 24 months of SSDI receipt, and here months of receipt are unlisted and therefore 0. The head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model treated SSDI income as qualifying the head for disability-based Medicare without the 24-month receipt requirement. Months of SSDI receipt default to 0, so the 57-year-old head fails that test and is not eligible." +us,scenario_073,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"The model assumed the 24-month SSDI waiting period was met for an ongoing disability, even though months of receipt are unlisted and therefore 0. It also misread the 'premiums excluding Medicare Part B' field as a sign of Medicare enrollment, when that field explicitly excludes Part B. The head is not eligible." +us,scenario_073,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model said PolicyEngine grants Medicare eligibility for any SSDI receipt under 65. PolicyEngine's disability pathway requires at least 24 months of SSDI receipt, and that input is unlisted and therefore 0, so the 57-year-old head is not eligible." +us,scenario_073,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disability,False,"The model treated SSDI income as enough to make an under-65 head Medicare eligible and never applied the 24-month SSDI receipt requirement. With months of receipt at 0, the age-65 test failed, and no ESRD, the head is not eligible." us,scenario_073,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no payroll-taxable wages and explicitly derived payroll tax of $0, but then submitted $2,846. Its numeric output contradicts its own payroll-tax-base reasoning and final stated derivation." -us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly passed the 200% FPL BBCE gross test and then applied the 100% FPL elderly/disabled net income test anyway. Because the household is categorically eligible, that net test is waived. Net income of about $1,757 gives a negative formula benefit, so the $24/month minimum allotment for a one-person household applies, which is $288 a year, not $0." -us,scenario_073,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income ($1,747) is above the 100% FPL net limit. Under Michigan BBCE (TANF non-cash), a categorically eligible household skips the net income test, so the negative formula benefit is raised to the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross limit and a $2,750 asset limit. It missed that Michigan's BBCE pathway uses a 200% FPL gross test, which the household passes at about 165% FPL. It also missed that this disabled household's $3,600 in assets passes the asset test. Categorical eligibility combined with a negative formula benefit yields the $24/month minimum, $288 a year." -us,scenario_073,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model's $0 matches stopping where 30% of net income (about $527) exceeds the $298 maximum allotment. It never applied the minimum allotment that a categorically eligible one-person household receives under Michigan BBCE: $24/month, $288 a year." -us,scenario_073,snap,claude-opus-4.7,llm_error,period_annualization,False,"The model treated about $5,200 of annual health premiums as a monthly medical deduction, which pushed net income below zero and produced the full $298 maximum. It also counted the pre-tax employer-sponsored premiums. The countable excess medical deduction is only about $182/month, so net income is $1,755. The formula benefit is negative, and the household gets only the $24 minimum ($288 a year)." -us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invoked the 130% FPL gross limit and the 100% FPL net limit to deny eligibility. Michigan's BBCE pathway uses a 200% FPL gross test, which the household passes, and it waives the net test. The categorically eligible single-person household therefore receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used a $587 maximum allotment, which is not the one-person figure; the one-person maximum is $298. It also put the $5,789 of pre-tax employer premiums into the medical deduction. Even with its understated $1,242 net income, 30% of that ($373) exceeds the $298 maximum, so the benefit falls to the $24/month minimum, $288 a year." -us,scenario_073,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income limit to the $1,747 net income and returned $0. A household categorically eligible under Michigan BBCE is exempt from the net test, so its negative formula benefit is raised to the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model used the 130% FPL gross test and a 100% FPL net test and found the household ineligible. Michigan BBCE grants categorical eligibility up to 200% FPL gross and does not apply a net income test. The household therefore receives the $24/month one-person minimum allotment, $288 a year." -us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model compared net income with a $1,255 net limit and concluded the 30% offset zeroes out the benefit. It ignored that a BBCE categorically eligible one-person household receives the minimum allotment ($24/month, $288 a year) whenever the formula benefit falls below it." -us,scenario_073,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model denied benefits because net income of about $1,750 exceeds the 100% FPL limit. Michigan's TANF non-cash BBCE pathway makes the household categorically eligible without a net test, and the $24/month minimum allotment applies, $288 a year." -us,scenario_073,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model correctly reached the minimum-allotment outcome but used $25/month instead of the 2026 minimum of $24/month (8% of the $298 one-person maximum), which gives $300 instead of $288. It also inflated the medical deduction by counting pre-tax employer-sponsored premiums." -us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the standard 130% FPL gross income test. It ignored Michigan's 200% FPL BBCE threshold, which the household passes at about 165% FPL, and the disabled-member exemption. The categorically eligible household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model denied benefits under a net income limit. A household categorically eligible through Michigan BBCE is not subject to the net test, and the $24/month one-person minimum allotment applies when 30% of net income exceeds the maximum, which totals $288 a year." -us,scenario_073,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model said gross and net income exceed SNAP limits. Gross income is about 165% FPL, which is under Michigan's 200% BBCE limit, and the categorically eligible household skips the net test. That yields the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly applied BBCE categorical eligibility and the minimum allotment but used the outdated $23/month minimum. The minimum in effect for 2026 is $24/month, so the annual total is $288, not $276." -us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said income exceeds the threshold, but gross income of about 165% FPL passes Michigan's 200% FPL BBCE test. The categorically eligible single-person household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the $298 maximum allotment but stopped at $0. A categorically eligible one-person household is guaranteed the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the maximum benefit and returned $0, without applying the $24/month minimum allotment owed to a categorically eligible one-person household ($288 a year). Its $1,541.50 net income also overstated deductions; the countable figure is $1,755." -us,scenario_073,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model claimed assets and income exceed SNAP limits. The $3,600 in assets passes the asset test, and gross income passes Michigan's 200% FPL BBCE test. The categorically eligible household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum benefit but used the outdated $23/month minimum. The 2026 minimum is $24/month, which gives $288 rather than $276." -us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model applied a net income cutoff. Michigan BBCE categorical eligibility removes the net test, and the household's negative formula benefit is raised to the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model applied the net income threshold and the 30% offset to reach $0. It missed that the household is categorically eligible under Michigan BBCE, so the net test does not apply and the $24/month minimum allotment does, $288 a year." -us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model computed a negative benefit and paid nothing, overlooking the minimum allotment for eligible one-person households: $24/month, $288 a year. It also inflated the medical deduction by counting $5,789 of pre-tax employer premiums; countable net income is $1,755, not $1,053." -us,scenario_073,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model compared annual gross income directly with the 100% FPL net limit, skipping both the deductions and categorical eligibility. Under Michigan BBCE the household passes the 200% FPL gross test, has no net test, and receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that income is above SNAP limits. Gross income of about 165% FPL is under Michigan's 200% FPL BBCE limit, so the household is categorically eligible and receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned $0 on the grounds that eligibility was not established, even though the prompt says to assume take-up. The household is categorically eligible through Michigan BBCE, and its negative formula benefit is raised to the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gpt-5.6-luna,llm_error,age_disability,False,"The model said no disabled status was listed, missing that SSDI receipt makes the head disabled for SNAP. It also applied a standard gross test instead of Michigan's 200% FPL BBCE test. The categorically eligible household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model concluded that the 30% benefit reduction leaves no positive allotment but did not apply the $24/month minimum allotment guaranteed to categorically eligible one- and two-person households, which gives $288 a year." -us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model stopped once the net income offset exceeded the maximum allotment. The BBCE categorically eligible one-person household receives the $24/month minimum allotment instead of $0, $288 a year." -us,scenario_073,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model correctly saw that no shelter deduction exists and that the formula benefit is not positive. It omitted the $24/month minimum allotment for a categorically eligible single-person household, which gives $288 a year." -us,scenario_073,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model computed zero support from roughly $2,146 of monthly income with no shelter costs and never applied the minimum allotment. The household is categorically eligible under Michigan BBCE and receives $24/month, $288 a year." -us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the net income limit to deny eligibility. Michigan BBCE categorical eligibility waives the net test, and the household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model correctly found that the 30% reduction fully offsets the maximum allotment but left the benefit at zero. A categorically eligible one-person household gets the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the one-person maximum allotment and returned $0, omitting the $24/month minimum allotment for categorically eligible one-person households ($288 a year). It also dropped the excess medical deduction available to this disabled member." -us,scenario_073,snap,grok-4.7,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test. Households categorically eligible through Michigan BBCE are exempt from that test, and the household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model found that 30% of net income exceeds the maximum allotment and returned $0 without applying the $24/month minimum allotment for a categorically eligible one-person household ($288 a year). It also overstated the medical deduction by including pre-tax employer-sponsored premiums. -us,scenario_073,snap,inkling,llm_error,thresholds_rates,False,"The model correctly applied the minimum-allotment rule against the $298 FY2026 maximum but used the outdated $23/month minimum. The FY2026 minimum is $24/month (8% of $298, rounded), which gives $288 instead of $276." -us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the one-person maximum but returned $0. The categorically eligible single-person household is entitled to the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net test, which BBCE categorical eligibility waives. It also omitted the excess medical deduction for the disabled member. With the household categorically eligible, the negative formula benefit is raised to the $24/month minimum, $288 a year." -us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model claimed the household exceeds both the gross and net limits. Gross income is under Michigan's 200% FPL BBCE threshold, and categorical eligibility removes the net test, so the household receives the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model wrongly applied a 100% FPL net test under BBCE and wrongly omitted the excess medical deduction for the SSDI-disabled member. A categorically eligible household is not subject to the net test, and its negative formula benefit is raised to the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross test with a misstated $1,311 limit. It ignored both the disabled-member gross test exemption and Michigan's 200% FPL BBCE threshold, which the household passes. Categorical eligibility then yields the $24/month minimum allotment, $288 a year." -us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model treated $3,600 in assets as over the resource limit. A household with a disabled member passes the asset test at that level, and the household is categorically eligible through Michigan BBCE. It therefore receives the $24/month minimum allotment, $288 a year." +us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly found net income of about $1,757 a month, then disqualified the household under the 100% FPL net income test. Michigan BBCE categorical eligibility waives that test. The model also never applied the one-person minimum allotment of $24 a month, which is $288 a year." +us,scenario_073,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It computed net income of about $1,747 and called the household ineligible under the roughly $1,305 net income limit. Michigan BBCE categorical eligibility exempts the household from the net test. The $24 monthly minimum for a one-person household therefore applies, giving $288 a year." +us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a 130% FPL gross limit and a $2,750 asset limit. Michigan BBCE uses a 200% FPL gross test, which $2,146 a month passes, and a categorically eligible household is not denied over $3,600 in assets. The household gets the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It returned $0 because 30% of net income wipes out the maximum allotment. It missed that a categorically eligible one-person household receives the $24 monthly minimum allotment, which is $288 a year." +us,scenario_073,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted about $5,165 of annual medical expenses from monthly income as though it were a monthly deduction. That drove net income to zero and produced the full $298 allotment. The correct monthly medical deduction leaves net income at $1,755, a $527 contribution, and only the $24 minimum ($288 a year)." +us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"It applied the 130% gross limit and the 100% FPL net income limit, then concluded the household was ineligible. Michigan BBCE categorical eligibility (200% FPL gross) removes the net test. That guarantees the $24 monthly minimum for a one-person household, or $288 a year." +us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"It used a maximum allotment of about $587. The one-person maximum is $298. It also inflated the medical deduction to about $700 a month by counting the $5,789 employer-sponsored premiums and double-counting the $2,600 premium. With net income of $1,755, the benefit falls to the $24 minimum, or $288 a year." +us,scenario_073,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It found net income of about $1,747 and denied the household under the 100% FPL net limit. Michigan BBCE makes the household categorically eligible, so the net test does not apply. The benefit is the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It applied the federal 130% gross and 100% net income tests and denied benefits. Michigan BBCE confers categorical eligibility at 200% FPL gross, so the net test is waived. The one-person minimum allotment of $24 a month, or $288 a year, applies." +us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"It correctly saw that 30% of net income exceeds the maximum allotment but set the benefit to $0. Categorically eligible one- and two-person households receive the minimum allotment of $24 a month, which is $288 a year." +us,scenario_073,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It computed net income of about $1,750 and denied the household under the 100% FPL net limit. Michigan BBCE categorical eligibility bypasses that test. The benefit therefore floors at the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly ended at the minimum allotment but used $25 a month. The minimum for a one-person household is $24 a month, which gives $288 rather than $300. It also overstated the medical deduction by including employer-sponsored premiums." +us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It failed the household on a 130% FPL gross test. That ignores Michigan's 200% FPL BBCE gross limit, which $25,754 a year passes. Categorical eligibility then yields the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It denied benefits because net income exceeds the one-person net income limit. Under Michigan BBCE the household is categorically eligible and exempt from the net test. It receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It treated income above the SNAP limits as producing no benefit. Michigan BBCE makes the household categorically eligible, and a one-person household then receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It correctly applied BBCE categorical eligibility and the minimum-benefit floor, but used the outdated $23 monthly minimum. The 2026 minimum is $24 a month, which gives $288 a year rather than $276." +us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It said countable income exceeds the one-person threshold. Michigan's 200% FPL BBCE gross limit is passed and the net test is waived. The household receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It correctly found that 30% of net income exceeds the maximum allotment but stopped at $0. A categorically eligible one-person household gets the $24 monthly minimum allotment, or $288 a year." +us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It computed a contribution of $462 that exceeds the maximum and returned $0, without applying the minimum allotment for categorically eligible one-person households ($24 a month, or $288 a year). Its net income of $1,541.50 also overstated the medical deduction; the correct figure is $1,755." +us,scenario_073,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It claimed assets and income exceed SNAP limits. Under Michigan BBCE the household passes the 200% FPL gross test, $3,600 in assets does not disqualify it, and it receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly identified BBCE eligibility with the minimum benefit but used the stale $23 monthly minimum. The 2026 minimum is $24 a month, so the annual amount is $288, not $276." +us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It treated net income above the benefit-producing level as yielding $0. A categorically eligible one-person household under Michigan BBCE receives the $24 monthly minimum allotment, or $288 a year." +us,scenario_073,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It applied a net income eligibility threshold and zeroed the benefit. Michigan BBCE waives the net test, and the one-person minimum allotment of $24 a month, or $288 a year, applies." +us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"It found the household eligible and the computed allotment negative, then paid $0 instead of the $24 monthly minimum that one- and two-person households receive ($288 a year). It also inflated medical expenses to $11,089 by counting employer-sponsored premiums and double-counting the $2,600 premium." +us,scenario_073,snap,glm-5.3,llm_error,categorical_eligibility,False,"It compared gross annual income to the 100% FPL net income limit and denied benefits. Michigan BBCE categorical eligibility (200% FPL gross, no net test) applies, and the household gets the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It declared income above the SNAP eligibility limits. At $2,146 a month the household passes Michigan's 200% FPL BBCE gross limit, is categorically eligible, and receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It wrongly said eligibility could not be established and returned 0. The facts establish Michigan BBCE categorical eligibility, with gross income below 200% FPL, so the $24 monthly minimum applies, giving $288 a year." +us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It treated the household as having no disabled member, even though the head receives SSDI, and then failed it on a gross income test. Michigan BBCE uses a 200% FPL gross limit, which the household passes, so it receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It concluded income is too high to produce a positive allotment and returned $0. Categorically eligible one-person households receive the $24 monthly minimum allotment, or $288 a year." +us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It returned $0 because counted income after deductions exceeds the amount that yields a benefit. It omitted the $24 monthly minimum for a categorically eligible one-person household, which is $288 a year." +us,scenario_073,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It returned $0 because no positive allotment remains after deductions. It never applied the $24 monthly minimum that categorically eligible one-person households receive, or $288 a year." +us,scenario_073,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It computed zero support from $2,146 of monthly income with no shelter costs. It skipped the $24 monthly minimum allotment owed to a categorically eligible one-person household, or $288 a year." +us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"It denied benefits under the net income limit for a one-person household. Michigan BBCE categorical eligibility exempts the household from the net test and yields the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,"It correctly found that the 30% reduction fully offsets the maximum allotment but returned $0. Categorically eligible one-person households receive the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,grok-4.6,llm_error,categorical_eligibility,False,"It returned $0 because 30% of net income exceeds the one-person maximum. It missed the $24 monthly minimum allotment for categorically eligible one- and two-person households, which is $288 a year." +us,scenario_073,snap,grok-4.7,llm_error,categorical_eligibility,False,"It applied the 100% of poverty net income test and denied benefits. Michigan BBCE categorical eligibility waives the net test, so the household receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It found that 30% of net income exceeds the maximum allotment and returned $0, omitting the $24 monthly minimum for a categorically eligible one-person household ($288 a year). It also overstated medical expenses by including employer-sponsored premiums." +us,scenario_073,snap,inkling,llm_error,thresholds_rates,False,"It correctly applied the minimum allotment for one- and two-person households but used the outdated $23 a month. The 2026 minimum is $24 a month, which gives $288 rather than $276." +us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It returned $0 because 30% of net income exceeds the one-person maximum. It missed the $24 monthly minimum allotment that categorically eligible one-person households receive, or $288 a year." +us,scenario_073,snap,kimi-k3,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit and zeroed the benefit, ignoring Michigan BBCE categorical eligibility and the $24 monthly minimum for one person ($288 a year). It also omitted the excess medical deduction, leaving net income at $1,942 instead of $1,755." +us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"It claimed income exceeds both the gross and net limits. Michigan's BBCE gross limit is 200% FPL, which the household passes, and categorical eligibility removes the net test. The benefit is the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,muse-wonder,llm_error,categorical_eligibility,False,"It denied benefits because net income exceeds the one-person net income limit. Under Michigan BBCE the household is categorically eligible without a net test and receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,ox-alpha,llm_error,categorical_eligibility,False,"It wrongly stated that Michigan BBCE still imposes a 100% FPL net income test, and it also dropped the excess medical deduction. Categorical eligibility waives the net test, so the household receives the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It refused to treat the SSDI recipient as disabled and applied a 130% FPL gross limit. Michigan BBCE uses a 200% FPL gross test, which $2,146 a month passes. The household gets the $24 monthly minimum, or $288 a year." +us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"It disqualified the household on $3,600 of resources. The household passes the asset test: $3,600 is below the $4,500 limit for households with an elderly or disabled member, and BBCE categorical eligibility also covers it. The $24 monthly minimum applies, or $288 a year." us,scenario_074,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_074,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_074,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model stated that tax-exempt SSDI counts in MAGI and then computed the opposite, summing only the $384 taxable IRA distribution plus a ""taxable portion"" of Social Security — applying the §86 provisional-income taxability test instead of the §36B(d)(2)(B) MAGI definition, which adds back the entire Social Security benefit excluded from gross income. Counting the full $33,640 SSDI plus the $384 IRA distribution puts the head at 2.13 x FPL, roughly 213% of poverty and far above Louisiana's 138% FPL expansion-adult limit." @@ -6019,34 +6089,35 @@ us,scenario_074,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eli us,scenario_074,head_medicaid_eligible,glm-5.3,llm_error,health_coverage,False,"The model excluded SSDI from MAGI on the reasoning that it is untaxable, reaching a MAGI of $3,410; Medicaid MAGI explicitly adds back Social Security benefits not included in gross income, so the $33,640 SSDI counts dollar-for-dollar. The correct MAGI of roughly $34,000 equals 2.13 x FPL, about ten times the model's figure and well above the ~$22,000 expansion-adult threshold it cited." us,scenario_074,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model asserted ""very low countable income"" with no derivation, a conclusion consistent only with dropping the entire $33,640 SSDI from countable income as non-taxable. Medicaid MAGI adds back untaxed Social Security benefits in full, yielding 2.13 x FPL — above Louisiana's 138% FPL expansion limit — and with age 54, zero SSI, and no dependents there is no non-MAGI category, so medicaid_category is NONE." us,scenario_074,head_medicaid_eligible,gpt-5.6-terra,llm_error,health_coverage,False,"The model restricted the MAGI base to ""the listed taxable income,"" which omits the untaxed SSDI that §36B(d)(2)(B) requires to be added back to AGI for Medicaid MAGI. Including the $33,640 SSDI with the $384 taxable IRA distribution gives 2.13 x FPL, so the head exceeds the 138% FPL expansion-adult limit and qualifies under no MAGI category." -us,scenario_074,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly saw that the head is under 65. It then assumed the 24-month SSDI waiting period was met because disability is 'constant for the full year.' But being constant this year says nothing about the 24 months before, and the unlisted months of SSDI receipt count as 0, so the disability route fails." -us,scenario_074,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model claimed that PolicyEngine grants Medicare whenever SSDI income is positive. PolicyEngine actually requires at least 24 months of SSDI receipt. That month count is unlisted and therefore 0, so a 54-year-old head with SSDI dollars is still ineligible." -us,scenario_074,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model reasoned that 'already receiving SSDI' means the 24-month waiting period has been met. The household gives no months of SSDI entitlement and that input defaults to 0, so the waiting period is not met and the age-65 test is the only route left, which the 54-year-old head fails." -us,scenario_074,head_medicare_eligible,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model named the 24-month benefit requirement and then treated it as met without any supporting fact. Months of SSDI receipt are unlisted and therefore 0, so the head fails the disability route and, at 54, fails the age-65 route too." -us,scenario_074,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model assumed the 24-month SSDI waiting period was met because status is held constant for the year. Year-long status does not supply the 24 months of prior entitlement, the unlisted month count is 0, and the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model wrongly read 'constant disability income status throughout the year' as meeting the 24-month SSDI requirement. The unlisted months of SSDI receipt are 0, so Medicare depends on age alone, and a 54-year-old head is not eligible." -us,scenario_074,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's disability rule is triggered by receiving SSDI benefits at all. The rule actually requires at least 24 months of SSDI receipt. That count is unlisted and therefore 0, so the under-65 head is not Medicare eligible." -us,scenario_074,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model saw that no SSDI start date was given and then assumed long-ongoing receipt. The prompt says to treat unlisted numbers as 0, so months of SSDI receipt are 0, the 24-month requirement fails, and the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model openly assumed the 24-month SSDI waiting period was met, which contradicts the instruction to treat unlisted inputs as 0. With zero months of SSDI receipt and an age of 54, the head meets neither Medicare route." -us,scenario_074,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model mentioned the disability waiting period but treated it as satisfied just because SSDI income is present. The months of SSDI receipt are unlisted and therefore 0, so the waiting period is not met and the 54-year-old head is not Medicare eligible." -us,scenario_074,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the disability route on the strength of the SSDI dollar amount alone and skipped the 24-month entitlement requirement. That requirement is unmet because months of SSDI receipt default to 0, so the under-65 head is ineligible." -us,scenario_074,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model treated receiving any SSDI income as automatic Medicare eligibility and never applied the 24-month SSDI receipt requirement. Months of receipt are unlisted and therefore 0, so the 54-year-old head is not eligible." -us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model equated being an SSDI recipient with being Medicare eligible and left out the 24-month entitlement requirement. Months of SSDI receipt are unlisted and therefore 0, so only the age-65 test applies, and the 54-year-old head fails it." -us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model gave Medicare eligibility based only on SSDI income and ignored the 24-month entitlement requirement. Months of SSDI receipt are 0 by default, so the 54-year-old head is not eligible." -us,scenario_074,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model said SSDI income alone qualifies the head for Medicare and skipped the 24-month waiting period. The unlisted month count is 0, so the waiting period is unmet, and at 54 the head fails the age-65 test." -us,scenario_074,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated receiving SSDI as sufficient for Medicare without checking that 24 months of entitlement had passed. That month count is 0, so the head qualifies only through age, and fails because they are 54." -us,scenario_074,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model gave Medicare eligibility directly from SSDI receipt and missed the 24-month entitlement requirement. Months of SSDI receipt are unlisted and therefore 0, so the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine treats anyone receiving SSDI as Medicare eligible regardless of age. PolicyEngine actually requires at least 24 months of SSDI receipt, which is unlisted and therefore 0, so the 54-year-old head is not eligible." -us,scenario_074,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly said age 54 does not qualify, but then concluded that the SSDI income does qualify, without applying the 24-month entitlement test. Months of SSDI receipt default to 0, so the disability route fails and the head is ineligible." -us,scenario_074,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt during the current year as satisfying the disability route. That route requires 24 months of prior SSDI entitlement, and the unlisted month count is 0, so the under-65 head is not Medicare eligible." -us,scenario_074,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine grants Medicare when SSDI income is positive. PolicyEngine actually tests months of SSDI receipt against a 24-month minimum. That input is unlisted and therefore 0, so the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied a made-up PolicyEngine rule triggered by positive SSDI dollars. The real test requires at least 24 months of SSDI receipt. That count defaults to 0 here, so the head fails the disability route and, at 54, the age-65 route too." -us,scenario_074,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed that 12 months of SSDI receipt in the year meets the 24-month waiting period. One year of receipt cannot meet a 24-month requirement, and the unlisted month count is 0 anyway, so the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model said receiving SSDI establishes Medicare eligibility 'after the statutory waiting period' and then assumed that period had already passed. Months of SSDI receipt are unlisted and therefore 0, so the waiting period is unmet and the under-65 head is not eligible." -us,scenario_074,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model concluded that a full year of SSDI makes the head a Medicare-eligible disabled beneficiary, but it ignored the 24-month entitlement requirement. That requirement is unmet because months of SSDI receipt default to 0, so the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model said the 24-month waiting period was 'assumed met since receiving ongoing disability benefits,' which adds a fact the prompt does not give. Months of SSDI receipt are 0 by default, so the under-65 head is not Medicare eligible." -us,scenario_074,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated SSDI income as qualifying for disability-based Medicare without applying the 24-month entitlement requirement. The unlisted month count is 0, so the requirement fails, and the 54-year-old head is ineligible." -us,scenario_074,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model mentioned the required waiting period and then counted it as met because facts are held constant for the year. Holding status constant this year does not supply 24 prior months of SSDI entitlement; that count is 0, so the 54-year-old head is not Medicare eligible." +us,scenario_074,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly named the 24-month SSDI waiting period but treated the year-long constancy instruction as proof it was met. Months of SSDI receipt is an unlisted input, which defaults to 0. The head, at 54, therefore meets neither the age-65 test nor the 24-month disability test." +us,scenario_074,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model claimed that SSDI income by itself confers Medicare eligibility under PolicyEngine. PolicyEngine actually requires at least 24 months of SSDI receipt, and that unlisted input defaults to 0. At 54, the head is not eligible." +us,scenario_074,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model reasoned that because the head is already receiving SSDI, the 24-month entitlement period must be satisfied. Current SSDI income says nothing about how long benefits have been paid, and the unlisted months-of-receipt input is 0. The disability pathway therefore fails, and the age-54 head is ineligible." +us,scenario_074,head_medicare_eligible,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model cited the 24-month rule but never checked whether it was met. With no SSDI duration given, months of receipt defaults to 0, so the 24-month test fails and the 54-year-old head is not Medicare-eligible." +us,scenario_074,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model read the 'constant year-long status' instruction as satisfying the 24-month waiting period. A constant status covers only the 12 months of the tax year, and the unlisted prior-receipt months default to 0. The 24-month test fails, and at 54 the head is ineligible." +us,scenario_074,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model acknowledged the 24-month SSDI requirement but treated constant SSDI income as meeting it. The months-of-receipt input is unlisted and therefore 0, so the head meets neither the age-65 nor the 24-month test." +us,scenario_074,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model asserted that PolicyEngine's disability rule makes any SSDI recipient eligible. That rule actually turns on at least 24 months of SSDI receipt, not on a positive benefit amount, and the unlisted months input defaults to 0. The head, at 54, is ineligible." +us,scenario_074,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model assumed ongoing SSDI with no start date means the 24-month waiting period is met. This contradicts the instruction to treat unlisted numeric inputs as 0: months of SSDI receipt is 0, so the disability pathway fails for the 54-year-old head." +us,scenario_074,head_medicare_eligible,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model explicitly assumed the 24-month SSDI waiting period was met, which infers an unlisted fact. With months of SSDI receipt at the default 0, the head is under 65 without qualifying disability duration and is not Medicare-eligible." +us,scenario_074,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model said the head qualifies 'after the disability waiting period' without checking that the period had passed. The unlisted months-of-receipt input is 0, short of the required 24, so the age-54 head is ineligible." +us,scenario_074,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model treated SSDI income as satisfying the disability pathway. That pathway requires at least 24 months of SSDI receipt, which defaults to 0 here because it is not listed. Since the head is also under 65, the answer is No." +us,scenario_074,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model equated positive SSDI income with Medicare eligibility. PolicyEngine requires at least 24 months of SSDI receipt, and that unlisted input is 0, so the 54-year-old head is ineligible." +us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated receiving SSDI as sufficient for Medicare and ignored the 24-month receipt requirement. With months of receipt at the default 0 and the head aged 54, no eligibility pathway is met." +us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model granted eligibility based on SSDI receipt alone. The disability pathway requires at least 24 months of SSDI receipt, which is unlisted and therefore 0, so the under-65 head is ineligible." +us,scenario_074,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model treated SSDI income itself as qualifying for Medicare and skipped the 24-month receipt test. That input defaults to 0, so the age-54 head fails both the age and disability-duration criteria." +us,scenario_074,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model concluded that receiving SSDI makes the head Medicare-eligible but never applied the 24-month receipt requirement. Months of receipt is an unlisted input that defaults to 0, so the 54-year-old head is ineligible." +us,scenario_074,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare eligibility and omitted the 24-month duration test. With that unlisted input at 0 and the head under 65, eligibility is No." +us,scenario_074,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine treats any SSDI recipient as Medicare-eligible regardless of age. PolicyEngine actually requires at least 24 months of SSDI receipt, which defaults to 0 here, so the 54-year-old head is ineligible." +us,scenario_074,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly found that age 54 fails the age test, then treated positive SSDI income as automatically qualifying. The disability pathway requires at least 24 months of SSDI receipt, and that unlisted input is 0, so the head is ineligible." +us,scenario_074,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt during the year as satisfying the disability pathway. That pathway requires at least 24 months of SSDI receipt, and the unlisted months input is 0, so the under-65 head is not Medicare-eligible." +us,scenario_074,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model asserted that PolicyEngine bases Medicare eligibility on positive SSDI income. PolicyEngine's disability test actually uses months of SSDI receipt (at least 24), which defaults to 0 here, so the 54-year-old head is ineligible." +us,scenario_074,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's disability pathway is triggered by positive SSDI income. It is triggered by at least 24 months of SSDI receipt, a separate input that defaults to 0 here, so the head fails the test and, at 54, also fails the age test." +us,scenario_074,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed that full-year SSDI receipt meets the 24-month waiting period. One year of receipt is at most 12 months, and the unlisted prior-receipt months default to 0, so the 24-month threshold is not met and the age-54 head is ineligible." +us,scenario_074,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model held that SSDI receipt establishes eligibility 'after the statutory waiting period' but never checked that the period had elapsed. Months of SSDI receipt is unlisted and therefore 0, below the 24 required, so the head is ineligible." +us,scenario_074,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model treated full-year SSDI receipt as conferring Medicare eligibility and ignored the 24-month receipt requirement. With the months-of-receipt input at its default of 0 and the head aged 54, eligibility is No." +us,scenario_074,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model explicitly assumed the 24-month waiting period was met because benefits are ongoing, inferring an unlisted fact. Months of SSDI receipt defaults to 0, so the disability pathway fails for the 54-year-old head." +us,scenario_074,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated SSDI income as qualifying for disability-based Medicare without applying the 24-month receipt requirement. That input is unlisted and defaults to 0, so the under-65 head is ineligible." +us,scenario_074,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disability,False,"The model treated SSDI income as opening the disability pathway despite the head being 54. The pathway requires at least 24 months of SSDI receipt, and that unlisted input is 0, so the head is not Medicare-eligible." +us,scenario_074,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model cited the waiting period, then assumed it was satisfied because facts are constant for the year. Year-long constancy gives at most 12 months, and the unlisted prior months of SSDI receipt default to 0, so the 24-month test fails and the age-54 head is ineligible." us,scenario_074,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,The model applied employee payroll-tax rates to non-wage income despite acknowledging that the household's income came from Social Security disability. It failed to set unlisted wages to zero and therefore invented a covered-earnings base that the household facts do not contain. us,scenario_074,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required output contract." us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning gets it right: $16,100 standard deduction, $13,843 ordinary tax plus $934 at 15%, totaling $14,777. It then submitted $12,523, a number with no derivation that contradicts its own calculation." @@ -6079,68 +6150,70 @@ us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_er us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It wrongly added the $2,240 tax-exempt pension to AGI, misadded the total to $131,866, and applied an over-65 standard deduction to a 62-year-old. It then reported $0 even though it computed $20,146 of tax and itself found no usable Credit for the Elderly or Disabled, so no credit justified zeroing the liability." us,scenario_075,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_075,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the listed disability as sufficient to activate a Florida disabled-adult Medicaid pathway without applying that pathway's complete eligibility requirements. The head receives no SSI and the engine assigns medicaid_category NONE, so no categorical pathway exists and Medicaid eligibility is 0." -us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"It treated the ""is disabled"" flag as automatic Medicare eligibility. The disability pathway requires 24 months of SSDI entitlement, and this head gets no Social Security disability benefits and has zero months of SSDI receipt. At 62, the head also fails the age-65 test, so the answer is not eligible." -us,scenario_075,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"It assumed disabled people under 65 are Medicare-eligible by default. Under-65 Medicare requires 24 months of SSDI entitlement or ESRD, and the head has neither: no Social Security disability income and no ESRD. With the head below the age-65 threshold, eligibility is 0." -us,scenario_075,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"It correctly named the 24-month SSDI requirement, then assumed the ""is disabled"" flag meets it, even though no Social Security disability benefits are listed. It also wrongly claimed that early retirement eligibility at 62 leads to Medicare; Social Security retirement at 62 never confers Medicare, and the Medicare age test is 65." -us,scenario_075,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"It equated being disabled with qualifying for Medicare. It skipped the requirement of 24 months of SSDI entitlement (or ESRD) for anyone under 65. The head receives no SSDI and is 62, so no pathway applies." -us,scenario_075,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"It asserted that PolicyEngine treats disabled individuals as Medicare-eligible. The rule actually requires age 65+, at least 24 months of SSDI receipt, or ESRD. This head has zero SSDI months and no ESRD, and is 62." -us,scenario_075,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"It invented an age-62 threshold for disability-based Medicare and treated ""approaching"" 65 as qualifying. Medicare requires age 65 or older, or 24 months of SSDI entitlement or ESRD; there is no age-62 pathway, and the head has no SSDI." -us,scenario_075,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"It applied the disability pathway based only on the ""is disabled"" flag. That pathway requires 24 months of Social Security disability benefit entitlement, and the head receives no SSDI. At 62, the head fails the age-65 test, so they are not eligible." -us,scenario_075,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"It claimed the head is Medicare-eligible at age 62, but the age threshold is 65. It also cited no valid alternative: the head has no months of SSDI receipt and no ESRD, so eligibility is 0." -us,scenario_075,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"It stated that anyone disabled is Medicare-eligible under 65. Under-65 eligibility actually requires at least 24 months of SSDI receipt or ESRD. The head has no Social Security disability benefits, so the disability flag alone does not qualify them." -us,scenario_075,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"It treated disability status as qualifying for Medicare under 65 and ignored the requirement of 24 months of SSDI entitlement. The head receives no SSDI, has no ESRD, and is 62, so they are not eligible." -us,scenario_075,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"It invoked the Medicare disability pathway based only on the ""is disabled"" flag. That pathway is triggered by 24 or more months of SSDI receipt, and this head has none. With no ESRD and age 62, the head is not eligible." -us,scenario_075,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"It claimed disability alone qualifies for Medicare. The modeled rule requires age 65+, at least 24 months of SSDI receipt, or ESRD, and the head meets none of these: age 62, no SSDI income, no ESRD." -us,scenario_075,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"It applied a disability-based Medicare rule keyed to the ""is disabled"" flag. The actual pathway requires 24 months of SSDI entitlement, and the head has zero SSDI months. With no ESRD and age below 65, eligibility is 0." -us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"It stated the rule as ""age 65+ or disabled"", dropping the requirement that under-65 disability eligibility rest on 24 months of SSDI receipt. The head receives no Social Security disability benefits, so only the age test applies, and it fails at 62." -us,scenario_075,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"It concluded that being 62 and disabled makes the head Medicare-eligible. Medicare needs age 65+, or 24 months of SSDI entitlement or ESRD for younger people. The head has no SSDI and no ESRD, so they are not eligible." +us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the 'is disabled' flag as enough for Medicare eligibility. The under-65 disability pathway requires 24 months of SSDI entitlement, and this head has no Social Security disability income at all. Because the head is 62 and has no SSDI receipt, ESRD or ALS, the head is not Medicare eligible." +us,scenario_075,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model claimed that being disabled alone makes a person under 65 Medicare eligible. It skipped the requirement to have been entitled to SSDI for 24 months. The head has no SSDI income (only wages and pensions), so the disability pathway fails and the age-65 test also fails." +us,scenario_075,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model named the 24-month SSDI requirement and then assumed it was met, even though the head has zero Social Security disability income. It also wrongly said that early retirement eligibility at 62 leads to Medicare, when Medicare's age threshold is 65. Neither pathway is met, so the head is ineligible." +us,scenario_075,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model stated that 'disabled individuals qualify for Medicare' without applying the requirement of 24 months of SSDI entitlement. The head receives no SSDI and is under 65, so the head does not qualify." +us,scenario_075,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model claimed that PolicyEngine treats any disabled person as Medicare eligible. In fact PolicyEngine's disability pathway requires at least 24 months of Social Security disability receipt, and this head has none. With the head aged 62, eligibility is No." +us,scenario_075,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented an age-62 threshold for disability-based Medicare and treated 'approaching' 65 as qualifying. Medicare needs either age 65 or 24 months of SSDI entitlement, and this head has neither. The head is not eligible." +us,scenario_075,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model applied the disability pathway based only on the disabled flag. That pathway requires 24 months of SSDI benefit entitlement, and the head has no Social Security disability income. The head is 62, so the age-65 test also fails." +us,scenario_075,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model asserted that the head is Medicare eligible at age 62, which misstates the age threshold of 65. It also gave no basis for a disability pathway. That pathway requires 24 months of SSDI receipt, and the head has none." +us,scenario_075,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model claimed that PolicyEngine treats any disabled person under 65 as Medicare eligible. PolicyEngine's rule requires at least 24 months of receiving Social Security disability benefits. The head has zero SSDI, so the head is ineligible." +us,scenario_075,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model let the disability status alone qualify a 62-year-old for Medicare. It skipped the requirement of 24 months of SSDI entitlement, which this head does not meet because no Social Security disability income is listed." +us,scenario_075,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied a 'modeled disability pathway' using only the disabled flag. That pathway depends on at least 24 months of SSDI receipt, and this head receives no SSDI. The head is under 65 and is not eligible." +us,scenario_075,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"The model treated being disabled as enough for Medicare. The actual rule requires 24 months of Social Security disability benefit entitlement for people under 65, and the head has no SSDI income. The correct value is 0." +us,scenario_075,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model invoked a disability-based Medicare rule without checking its qualifying condition, which is at least 24 months of SSDI receipt. The head's income contains no Social Security disability benefits, so the pathway fails and the head, at 62, is ineligible." +us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model stated the rule as 'age 65+ or disabled'. PolicyEngine's actual rule is age 65+, 24 or more months of SSDI receipt, or ESRD/ALS. The head is disabled but receives no SSDI, so the head is not Medicare eligible." +us,scenario_075,head_medicare_eligible,muse-wonder,llm_error,age_disability,False,"The model applied the under-65 disability pathway based only on the disabled flag. That pathway requires 24 months of entitlement to Social Security disability benefits, and this head has none, so the head is ineligible." +us,scenario_075,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model concluded that being 62 and disabled makes the head Medicare eligible. Under 65, Medicare eligibility requires 24 months of SSDI entitlement (or ESRD/ALS), and the head has no Social Security disability income." us,scenario_075,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"The model's explanation applies 6.2% Social Security plus 1.45% Medicare to the $12,656 in wages and ends with 'value = 968', which is the correct derivation. The recorded answer is $0.00, so the numeric value field did not match the explanation's 968. This is a submission/parse failure, not a tax error." us,scenario_075,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly limited the base to $12,656 in wages, used 6.2% plus 1.45%, and excluded Additional Medicare Tax. It then miscalculated the total: $784.67 + $183.51 = $968.18, not the $966 it reported." us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model used the right base ($12,656 in wages) and the right rates (6.2% Social Security plus 1.45% Medicare), which yield $968.18. Its $1,054 implies an effective rate of about 8.33%, so it miscalculated the rate-times-base step." us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax. The correct derivation is 7.65% of $12,656 in wages, which is about $968." us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed $784.67 of Social Security tax plus $183.51 of Medicare tax, for a total of $968.18. It then reported $989, citing 'rounding', which contradicts its own correct total." us,scenario_076,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model falsely extended Idaho WIC eligibility through age 8 even though WIC child eligibility ends at the fifth birthday. It also divided or expressed the income ratio incorrectly: using its own $23,030 poverty guideline, $173,820 is about 755% of poverty and exceeds the 185% limit by a wide margin; medical expenses do not establish WIC eligibility or cure either failure." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The setup was nearly right, with taxable income of about $139,120 and bracket tax of about $24,321. The model then swapped in an unexplained 'tentative tax' of $29,226, inflating the result by about $5,000. Using the exact 2026 head-of-household thresholds on $139,270 gives $24,211.80, and subtracting $4,600 in credits leaves $19,611.80." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"AGI, the $24,150 deduction, the $800 QBI deduction, $139,270 of taxable income and the $4,600 in credits were all correct. The model then used 2025 head-of-household brackets (10% to $17,000, 12% to $64,850, 22% to $103,350) instead of the 2026 thresholds ($17,700 / $67,450 / $105,700). That overstated bracket tax by $321 ($24,532.80 instead of $24,211.80)." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model added the nontaxable $9,600 of child support to gross income and subtracted the $6,480 ESI premium from wages. It used a made-up $26,050 standard deduction and brackets with no 24% rate, applied a $2,000-per-child CTC instead of $2,200, and left out the QBI deduction and the $200 CDCC. Its $18,863 is the product of these offsetting errors." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"The model gave only a rough estimate of $30,500, which is higher than even the correct pre-credit tax of $24,211.80. That means it never properly applied the $24,150 standard deduction, the $800 QBI deduction, the 2026 head-of-household brackets, or the $4,600 of nonrefundable CTC and CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated mortgage interest (about $29,400) from the $420,000 balance and added an estimated SALT amount so it could itemize. The prompt says not to infer unlisted expenses, so mortgage interest is $0 and the $24,150 standard deduction applies. It also used 2025 brackets and a $2,000-per-child CTC, then replaced its own computed $17,033 with an unexplained $21,889." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model estimated about $25,200 of mortgage interest from the loan balance plus an estimated state tax deduction, so it itemized $33,200. Mortgage interest is $0 under the facts given, so the $24,150 standard deduction applies. It also used a $2,000-per-child CTC and then dropped its computed $14,306 for an unsupported $21,625." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model treated $1,700 per child of the CTC as refundable and so applied only about $1,000 of nonrefundable CTC. With roughly $24,000 of liability, the full $4,400 CTC is used nonrefundably. It also used a $24,500 standard deduction and 2025 brackets instead of the 2026 values ($24,150 and thresholds of $17,700 / $67,450 / $105,700)." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the §199A QBI deduction of $800 (20% of the $4,000 qualified rental income). That put taxable income at $140,070 instead of $139,270 and overstated tax by $192 (24% × $800). Everything else, including the 2026 brackets and the $4,600 in credits, was correct." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model filed the unmarried parent of two qualifying children as single instead of head of household and used 2025 single brackets. It itemized $37,300 by estimating $27,300 of mortgage interest from the balance plus $10,000 of SALT, although the prompt lists no interest or tax payments. It also used $2,000 per child for the CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used single status (a $16,100 deduction and single brackets) instead of head of household with the $24,150 deduction. It also subtracted the $6,480 ESI premium from the stated gross wages and used $2,000 per child for the CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the $800 QBI deduction on the rental income, so it taxed $140,070 instead of $139,270. It also used 2025 head-of-household brackets, giving $24,724.80 instead of the $24,211.80 that the 2026 thresholds produce on $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"The model claimed the CTC is fully refundable in 2026 and so applied $0 of it nonrefundably. In fact the $4,400 CTC first offsets income tax and is fully used here. It also used a $23,000 deduction with no QBI deduction and overstated bracket tax at $27,259 instead of $24,211.80." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated $160,000 gross wages, giving an AGI of $157,740 instead of $164,220. It also used a $23,200 deduction instead of $24,150, omitted the $800 QBI deduction, and used a $2,000-per-child CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026 and applied a pre-TCJA standard deduction, three personal exemptions and old brackets, and it treated the CTC as fully phased out. 2026 law keeps the $24,150 head-of-household deduction, no personal exemptions and a $2,200-per-child CTC. It also subtracted the ESI premium from wages, and its submitted $26,346.25 does not even match its own $24,775.75." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026, using a $11,687 deduction, personal exemptions and a $1,000-per-child CTC. 2026 law keeps the $24,150 head-of-household deduction, no exemptions, the 10/12/22/24% brackets and a $2,200-per-child CTC. It also subtracted the $6,480 ESI premium from gross wages." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied a $2,000-per-child CTC ($4,000) instead of the 2026 amount of $2,200 ($4,400) and left out the $200 CDCC. Its $18,985 implies pre-credit tax of $22,985, about $1,227 below the $24,211.80 that the 2026 head-of-household brackets give on $139,270 after the $800 QBI deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expires in 2026, applying personal exemptions, a $12,090 deduction and pre-TCJA brackets, and it allowed no CTC. 2026 law keeps the $24,150 deduction, the 10/12/22/24% brackets and a $4,400 CTC that is fully nonrefundable here. It also subtracted the ESI premium from the stated gross wages." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied TCJA-expiration ('sunset') law, with personal exemptions, pre-TCJA brackets and a CTC fully phased out at a $75,000 threshold. 2026 law gives a $2,200-per-child CTC that phases out above $200,000, so the full $4,400 applies. It also subtracted the $6,480 ESI premium from gross wages." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $19,535 implies pre-credit tax of $24,135 after the same $4,600 in credits, which is $76.80 below the exact result. Applying the 2026 head-of-household thresholds ($17,700 / $67,450 / $105,700) to $139,270 gives $24,211.80, so the model approximated the bracket tax instead of computing it exactly." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reduced income for 'pre-tax health insurance deductions', but the $160,000 is stated as gross wages and the engine's AGI is $164,220. Its $21,743 is $2,131 too high, which means it did not apply the full $4,400 CTC plus $200 CDCC against the $24,211.80 bracket tax on $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model treated the CTC as phased out, which follows from assuming the TCJA expires, and applied only the $200 CDCC. In 2026 the CTC is $2,200 per child with a $200,000 phase-out threshold, so the full $4,400 is nonrefundable. It also subtracted the ESI premium from wages, giving an AGI of $157,740 instead of $164,220." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used outdated 2025 parameters: a $22,500 head-of-household deduction instead of $24,150, 2025 brackets, and a $2,000-per-child CTC instead of $2,200. It also left out the $800 QBI deduction on the rental income." -us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated gross wages, giving an AGI of $157,740 instead of $164,220. It used a $23,100 deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200, and its errors offset to $19,279.60." -us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $23,625 deduction (the 2025 figure) instead of $24,150 and a $2,250-per-child CTC instead of $2,200. Its bracket tax of $24,965 on $139,795 is overstated; the 2026 head-of-household thresholds give $24,211.80 on the correct $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no figures. Its $14,143 implies pre-credit tax of about $18,743, roughly $5,470 below the $24,211.80 that 2026 head-of-household brackets give on $139,270. That points to an overstated deduction (such as itemizing estimated mortgage interest) or joint-width brackets in place of the $24,150 standard deduction and head-of-household schedule." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model counted the $9,600 of child support as taxable income; child support is excluded from gross income. It also did not subtract the $4,400 CTC and $200 CDCC from the $24,211.80 bracket tax on $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated gross wages, giving an AGI of $157,740 instead of $164,220. It also applied a $2,000-per-child CTC ($4,000) instead of the 2026 amount of $2,200 ($4,400)." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status, with the $32,200 deduction and joint brackets, for an unmarried parent who files as head of household. Head-of-household status uses the $24,150 deduction and narrower brackets. It also left out the $800 QBI deduction." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"The method was right (head of household, standard deduction, $4,600 in credits), but the answer is $5.80 low. That means the bracket tax was rounded to about $24,206 instead of computed exactly. The exact 2026 head-of-household bracket computation on $139,270 gives $24,211.80." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model applied personal exemptions, which follows from assuming the TCJA expires in 2026. 2026 law has no personal exemptions, a $24,150 head-of-household deduction, the 10/12/22/24% brackets and a $2,200-per-child nonrefundable CTC." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $23,063 deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200. Its bracket tax of about $24,570 overstates the $24,211.80 that 2026 head-of-household brackets give on $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied TCJA-expiration law: a $12,450 deduction, $15,900 of personal exemptions, pre-TCJA brackets and a CTC phased out above $75,000. 2026 law keeps the $24,150 deduction, no exemptions, the 10/12/22/24% brackets and a $2,200-per-child CTC that phases out only above $200,000." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied TCJA-expiration law: $5,300 exemptions, a $12,250 deduction, 10/15/25% brackets and a $1,000 CTC fully phased out. 2026 law keeps the $24,150 deduction, the 10/12/22/24% brackets and the full $4,400 CTC, which is nonrefundable here." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied TCJA-expiration law, using personal exemptions, a $12,585 deduction, 2017-style brackets and no CTC. 2026 law keeps the $24,150 deduction and a $4,400 CTC. It also subtracted the $6,480 ESI premium from the stated gross wages." -us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model applied only about $1,000 of nonrefundable CTC, treating the rest as refundable. With $24,211.80 of liability, the full $4,400 CTC is used nonrefundably, along with the $200 CDCC. Its rounded bracket tax of about $24,500 also overstates the exact figure." -us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was nothing to score." -us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model got AGI, the $24,150 deduction, the $800 QBI deduction, $139,270 of taxable income and the $4,600 in credits right. It then reported bracket tax of $24,332.80, which is $121 above the $24,211.80 that the 2026 head-of-household thresholds ($17,700 / $67,450 / $105,700) give." -us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a $1,815 depreciation deduction ($49,921 / 27.5) against rental income. The unadjusted basis of qualified property is only a §199A limit input, so rental income stays $4,000 and the QBI deduction is $800. It also used a $23,200 deduction instead of $24,150 and $2,000 per child for the CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model split the CTC into $3,400 refundable and $1,000 nonrefundable. The full $4,400 CTC is first applied against the $24,000+ liability and is entirely nonrefundable here. It also left out the $800 QBI deduction, taxing $140,070 instead of $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status with a made-up $16,600 deduction for an unmarried parent who files as head of household (deduction $24,150). It also left out the $800 QBI deduction and used $2,000 per child for the CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model counted nontaxable child support in AGI and used an invented $24,800 deduction and brackets. It declared the CTC fully phased out using a $400,000 threshold that its own $169,820 AGI does not exceed. It then submitted $18,779.30, which does not match its computed $14,152.40." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly found AGI of $164,220, the $800 QBI deduction and roughly $24,321 of bracket tax. It then swapped in an unexplained 'tentative tax of $29,226', adding about $4,700, and subtracted credits from that. The correct path is $24,211.80 minus $4,600 in nonrefundable credits." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model had the right taxable income ($139,270) and the right credits ($4,600). However, it applied the 2025 HoH bracket thresholds ($17,000/$64,850/$103,350) instead of the 2026 thresholds ($17,700/$67,450/$105,700). That overstated pre-credit tax by $321 ($24,532.80 vs $24,211.80)." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model counted the nontaxable $9,600 of child support as income and subtracted the $6,480 ESI premium. It also used a made-up $26,050 standard deduction and incorrect brackets, applied a $2,000-per-child CTC instead of $2,200, and left out the $200 CDCC and the $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"The model gave a rough $30,500 without doing any bracket math. The correct calculation is $139,270 of taxable income, $24,211.80 of tax at 2026 HoH rates, then minus the $4,400 CTC and $200 CDCC. The model's figure is about $10,900 too high, so it did not net the credits or use HoH parameters correctly." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented about $29,400 of mortgage interest from the loan balance alone, although no interest paid was listed. It also imputed state income tax, then itemized instead of taking the $24,150 HoH standard deduction. It used a $2,000 CTC per child instead of $2,200, and its submitted $21,889 contradicts its own computed $17,033." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model imputed $25,200 of mortgage interest from the $420,000 balance (no interest paid was listed) and itemized instead of taking the $24,150 standard deduction. It used a $2,000 CTC per child instead of $2,200. It then submitted $21,625, which does not match its own computed $14,306." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model claimed only about $1,000 of the $4,400 CTC was nonrefundable, treating the $1,700-per-child refundable cap as carved out of the credit. In fact the whole CTC is nonrefundable when tax exceeds it, and the refundable ACTC only covers any unused remainder. It also used approximate 2025-style brackets and a $24,500 standard deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the $800 Section 199A QBI deduction on the $4,000 of rental income, so taxable income came out at $140,070 instead of $139,270. At the 24% marginal rate, that overstated tax by exactly $192." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model filed the unmarried parent of two dependent children as single instead of head of household, and used the single brackets. It also imputed $27,300 of mortgage interest and $10,000 of SALT to itemize, instead of taking the $24,150 HoH standard deduction. It used a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used the single standard deduction and single brackets instead of head of household. It also subtracted the $6,480 ESI premium from the listed gross wages, which cut AGI to $157,740 instead of $164,220. It used a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the $800 QBI deduction on the rental income, giving taxable income of $140,070 instead of $139,270. It then applied 2025 HoH bracket thresholds, producing $24,724.80 of tax instead of $24,211.80." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"The model wrongly declared the CTC fully refundable in 2026 and set the nonrefundable CTC to $0. In fact the $4,400 CTC is nonrefundable up to tax liability. It also used a $23,000 standard deduction, left out the $800 QBI deduction and overstated bracket tax at $27,259." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the stated gross wages, putting AGI at $157,740 instead of $164,220. It used a $23,200 standard deduction instead of the 2026 HoH $24,150 and left out the $800 QBI deduction. It applied a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, bringing back personal exemptions, a $12,155 standard deduction and pre-TCJA brackets, and treated the CTC as fully phased out. Under the OBBBA, the 2026 law keeps the $24,150 HoH standard deduction and a $2,200 CTC. It also subtracted the ESI premium, and its submitted $26,346.25 contradicts its own computed $24,775.75." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a TCJA sunset: an $11,687 standard deduction, three personal exemptions, pre-TCJA brackets and a $1,000-per-child CTC. The correct 2026 parameters are the $24,150 standard deduction, the 10/12/22/24 brackets and a $2,200 CTC. It also subtracted the $6,480 ESI premium from gross wages." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied a $2,000-per-child CTC ($4,000) instead of $2,200 ($4,400) and did not subtract the $200 CDCC. Its implied pre-credit tax of about $22,985 is below the $24,211.80 that the 2026 HoH brackets give on $139,270 of taxable income." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, using a roughly $12,090 standard deduction, personal exemptions and pre-TCJA brackets, and allowed no CTC at all. The 2026 law gives a $24,150 HoH standard deduction and a $4,400 nonrefundable CTC. It also subtracted the ESI premium from gross wages." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used 'sunset' 2026 law (personal exemptions, pre-TCJA brackets) and treated the CTC as phased out by high income. Under the 2026 law the $2,200-per-child CTC starts phasing out at $200,000, so the full $4,400 applies. It also subtracted the ESI premium from AGI." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. The correct path is $24,211.80 of tax on $139,270 at 2026 HoH rates, minus $4,600 of credits, which gives $19,611.80. Its $19,535 is $76.80 lower, so it used pre-credit tax of about $24,135 from bracket or deduction parameters that are not the exact 2026 HoH ones." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model said it computed income 'after pre-tax health insurance deductions', subtracting the $6,480 ESI premium from wages that were stated as gross. AGI is $164,220, not $157,740. Its $21,743 also does not reflect the full $4,400 CTC plus $200 CDCC taken off $24,211.80." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model treated the CTC as phased out. Under 2026 law the $2,200-per-child CTC only phases out above $200,000, so the full $4,400 applies at $164,220 AGI. It also subtracted the ESI premium from gross wages, getting AGI of $157,740." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $22,500 HoH standard deduction and 2025 bracket thresholds instead of the 2026 $24,150 deduction and brackets. It left out the $800 QBI deduction and applied a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from the listed gross wages, putting AGI at $157,740 instead of $164,220. It used a $23,100 standard deduction instead of $24,150 and applied a $2,000-per-child CTC instead of the 2026 $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $23,625 HoH standard deduction instead of the 2026 $24,150 and inflated bracket tax to $24,965, against $24,211.80 on the correct taxable income. It applied a $2,250-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no computation. The correct derivation yields $19,611.80 ($24,211.80 at 2026 HoH rates on $139,270, less $4,600 in credits). Its $14,143 is about $5,470 too low, which matches using a much larger deduction or the wider joint brackets instead of the HoH standard deduction and HoH brackets." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the $9,600 of child support as taxable income, but child support is excluded from gross income. It also did not net the full $4,400 CTC and $200 CDCC against tax, which left the answer about $9,300 too high." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,480 ESI premium from wages that were given as gross, getting AGI of $157,740 instead of $164,220. It applied a $2,000-per-child CTC ($4,000) instead of the 2026 $2,200 per child ($4,400)." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model filed as married filing jointly with a $32,200 standard deduction and joint brackets, even though the head is unmarried and qualifies for head of household ($24,150 deduction, HoH brackets). It also left out the $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model gave $19,606, which is $5.80 below the exact result. Correctly, $24,211.80 of tax at 2026 HoH rates on $139,270, minus $4,600 of credits, is $19,611.80, so the model rounded or slightly misapplied the bracket computation instead of carrying the exact cents." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's 'deductions and exemptions' wording shows it brought back personal exemptions under an assumed TCJA sunset. The 2026 law has no exemptions, a $24,150 HoH standard deduction, the 10/12/22/24 brackets and a $4,400 nonrefundable CTC. This left the result about $3,600 too high." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $23,063 standard deduction instead of the 2026 HoH $24,150 and overstated bracket tax at $24,570 instead of $24,211.80. It applied a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired: a $12,450 standard deduction, $5,300 exemptions, 10/15/25 brackets, and a $1,000 CTC phased out above $75,000. Under the OBBBA, 2026 keeps the $24,150 HoH deduction and a $2,200 CTC with a $200,000 phase-out, so the full $4,400 is allowed." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied post-TCJA-expiration law: personal exemptions, a $12,250 standard deduction, 10/15/25 brackets, and a $1,000 CTC phased out above $75,000. The 2026 law gives a $24,150 deduction, the 10/12/22/24 brackets and a full $4,400 CTC at this income." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, using a $12,585 standard deduction, personal exemptions, inflation-adjusted 2017 brackets and no CTC, instead of the 2026 $24,150 deduction and the $4,400 nonrefundable CTC. It also subtracted the $6,480 ESI premium from gross wages." +us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model counted only about $1,000 of the CTC as nonrefundable. The whole $4,400 CTC is nonrefundable when tax liability ($24,211.80) exceeds it, and ACTC only applies to any unused excess. Leaving out the remaining $3,400 put the answer about $3,700 too high." +us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for federal_income_tax_before_refundable_credits, so the answer is missing and cannot be scored." +us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model had the right taxable income ($139,270) and credits ($4,600), but its bracket tax of $24,332.80 does not match the 2026 HoH schedule (10% to $17,700, 12% to $67,450, 22% to $105,700, 24% above). That schedule gives $24,211.80, so the answer is $121 too high." +us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented $1,815 of depreciation from the unadjusted property basis and cut rental income to $2,185, but rental income is taken as the $4,000 given, which yields an $800 QBI deduction. It also used a $23,200 standard deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model had the right taxable income ($139,270) and credits ($4,600), but it computed tentative tax of $24,329 instead of the $24,211.80 that the 2026 HoH brackets give. It used bracket thresholds that are not the 2026 HoH ones." +us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model split the $4,400 CTC into $3,400 refundable and $1,000 nonrefundable before comparing it with tax liability. In fact the whole CTC is nonrefundable up to the $24,211.80 tax, and the refundable cap only applies to any unused remainder. It also left out the $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly brackets and an invented $16,600 standard deduction for an unmarried head of household. The correct parameters are the HoH $24,150 deduction and HoH brackets. It also left out the $800 QBI deduction and used a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model counted nontaxable child support in AGI and subtracted made-up $1,700 dependent amounts. It claimed the CTC was phased out above $400,000 even though AGI is about $170,000, which is wrong. It then submitted $18,779.30, which does not match its own computed $14,152.40 and skips the correct path of $24,211.80 minus $4,600." us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied the married-filing-jointly $400,000 phase-out threshold and a $2,000-per-child CTC instead of the head-of-household $200,000 threshold and 2026's $2,200 per child, then — after correctly stating that ACTC equals the CTC in excess of tax liability — ignored that offset even though pre-credit tax above $25,000 absorbs the entire $4,400 CTC nonrefundably. Its $2,400 is not the output of any rule it named; it is an unexplained reduction of the $3,400 ACTC ceiling, while the excess-over-liability computation yields $0." us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It stated the governing rule — the refundable ACTC is 'claimed to the extent CTC exceeds tax used' — and then contradicted it by reporting the $1,700-per-child cap of $3,400 as the refundable amount. With taxable income near $140,000 and pre-credit federal tax over $25,000, the full $4,400 CTC is consumed as a nonrefundable credit, leaving zero excess to convert into refundable_ctc." us,scenario_076,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It treated the entire Child Tax Credit as refundable, calling it a 'refundable child tax credit of 2,000 per child,' which errs twice: the 2026 per-child credit is $2,200, and only the portion unused against tax liability is refundable, capped at $1,700 per child. Because this filer's federal tax before credits exceeds $25,000, all $4,400 of CTC is applied nonrefundably and the refundable component is $0, not $4,000." @@ -6161,93 +6234,95 @@ us,scenario_076,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model us,scenario_076,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_076,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $4,000 of rental income as earnings from self-employment and invented a 30% expense deduction that the prompt did not provide. Rental income is not subject to self-employment tax here, leaving a zero tax base and $0 liability." us,scenario_076,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no self_employment_tax output or explanation, violating the required structured-output contract." -us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It reached $139,270 of taxable income, so it missed the $1,000 Idaho child-care subtraction. It then applied 5.3% to the full amount with no HoH zero-rate band of about $9,622. It also subtracted a $410 child tax credit that the 2026 computation does not apply, left out the $10 permanent building fund tax, and finally pushed the result up to $7,099 for no stated reason." -us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It took the $1,000 child-care subtraction but left out the $800 QBI deduction, giving $139,070 instead of $138,270. It then applied 5.3% to the whole amount instead of only the income above the roughly $9,622 HoH zero-rate band. It also subtracted a $410 child tax credit that is not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It counted non-taxable child support as income and used the single $15,000 standard deduction instead of the $24,150 HoH deduction. It also added an invented $9,380 adjustment and used made-up graduated brackets instead of Idaho's flat 5.3% rate above the zero-rate band. Together these push taxable income to $161,720 and the tax to $9,400." -us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It gave a rough $5,800 from a 5.3% approximation minus 'dependent credits', with no computation shown. The correct steps are 5.3% × ($138,270 − ~$9,622) = $6,818.34, with no Idaho nonrefundable credit, plus the $10 permanent building fund tax. Its figure is consistent with subtracting credits and deductions that do not apply." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It used the repealed 5.695% rate instead of the 2026 rate of 5.3%, and the 2025 standard deduction of $23,625. It left out the $800 QBI deduction and the $1,000 child-care subtraction. It then taxed nearly the full $140,595 and raised the result to $8,204 with no supporting arithmetic." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It used the outdated 5.695% rate and an understated $22,500 HoH standard deduction instead of $24,150. It skipped the QBI and child-care subtractions and used a $3,420 zero band instead of about $9,622. It then inflated its own $7,876 figure to $9,112 without a basis." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It applied 5.3% to about $139,720 with no HoH zero-rate band of about $9,622. It used a $24,500 standard deduction instead of $24,150 and left out the $800 QBI deduction and $1,000 child-care subtraction. It then raised the result above its own $7,405 figure instead of computing $6,818.34 + $10." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It used taxable income of $140,070, leaving out the $800 QBI deduction and the $1,000 child-care subtraction that bring it to $138,270. It used the unindexed $9,346 zero band instead of the 2026 HoH figure of about $9,622. It also subtracted a $410 child tax credit that is not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It filed as single with a $15,000 standard deduction instead of head of household with $24,150. It also deducted three $4,714 personal exemptions, which Idaho no longer allows, and used the repealed 5.695% rate instead of 5.3% above the zero-rate band. On top of that it subtracted a $100 child-care credit." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It used the outdated 5.695% rate on nearly all income and a $16,000 standard deduction instead of the $24,150 HoH deduction. It skipped the QBI and child-care subtractions and the zero-rate band. It then inflated its own $8,443 figure to $9,400 with invented adjustments." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied 5.3% to all of $140,070, skipping the HoH zero-rate band of about $9,622, the $800 QBI deduction and the $1,000 child-care subtraction. It then subtracted a $410 child tax credit that is not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It used the 2022 rate of 5.8% instead of the 2026 rate of 5.3%, and a $19,900 HoH standard deduction instead of $24,150. It also left out the QBI and child-care subtractions, the zero-rate band of about $9,622, and the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It cut AGI to $157,740 by subtracting the $6,480 employer insurance premiums from wages that are already gross, when AGI is $164,220. It used a $23,200 standard deduction instead of $24,150 and the outdated 5.8% rate instead of 5.3% above the zero-rate band. It also left out the QBI and child-care subtractions." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted the $6,480 insurance premiums to get a $157,740 AGI instead of $164,220. It used a $12,155 HoH standard deduction instead of $24,150 and applied the repealed 5.695% rate to the full amount with no zero-rate band. It also submitted $8,247.54, which does not match its own $8,291.07." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a wrong 5.0% flat rate on $140,070 instead of 5.3% on income above the roughly $9,622 HoH zero band. It skipped the $800 QBI and $1,000 child-care subtractions. It also subtracted $450 of child tax credit and CDCC that the Idaho 2026 computation does not apply, and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used an understated $130,867 taxable income instead of $138,270 and the outdated 5.8% rate instead of 5.3%. It skipped the zero-rate band, subtracted a $410 child tax credit that is not applied, and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation. $8,100 matches applying an outdated 5.695–5.8% rate to roughly $140,000 with no zero-rate band. The correct result is 5.3% × ($138,270 − ~$9,622) + $10 = $6,828.34." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It used the repealed 5.695% rate instead of 5.3%, applied it with no zero-rate band, and used an understated taxable income of $129,750 instead of $138,270. It also left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It used the outdated 5.8% rate instead of the 2026 rate of 5.3%, ignored the HoH zero-rate band of about $9,622, and subtracted child tax credits that the 2026 computation does not apply. This overstates the tax at $8,043." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation. $7,322 is consistent with applying 5.3% to roughly $138,000–$139,000 with no zero-rate band. The correct steps exclude the first ~$9,622 of HoH income from tax, which gives $6,818.34, and then add the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"$6,985 matches 5.3% on about $139,000 of taxable income with no zero-rate band, minus a $410 child tax credit. The correct steps exempt the first ~$9,622 of HoH income, apply no nonrefundable credit, and add the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation. $7,665 overstates the tax and is consistent with a flat rate above 5.3% on nearly all taxable income with no zero-rate band of about $9,622. The correct result is 5.3% × $128,648 + $10 = $6,828.34." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It wrongly claimed Idaho income tax is not modeled and returned $0. The household owes Idaho tax of 5.3% on taxable income of $138,270 above the HoH zero-rate band, plus the $10 permanent building fund tax, for $6,828.34." -us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It used the repealed 5.695% rate on all of $141,720 with no zero-rate band and an understated $22,500 standard deduction instead of $24,150. It also left out the $800 QBI deduction and $1,000 child-care subtraction." -us,scenario_076,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It started from federal taxable income, which already reflects the standard deduction, and then subtracted a $23,625 Idaho standard deduction again. That takes taxable income down to $116,170 instead of $138,270. It also subtracted a $410 child tax credit that is not applied and ignored the zero-rate band." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave no computation. $6,494 understates the result: 5.3% on the $128,648 above the HoH zero-rate band gives $6,818.34, plus the $10 permanent building fund tax. Its figure is consistent with subtracting credits or extra deductions that do not apply." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It referred to itemized medical deductions, which do not apply because the $24,150 standard deduction is used. $7,450 is consistent with taxing roughly $140,000 at 5.3% with no zero-rate band of about $9,622." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It applied 5.3% to all of $140,870 using a $23,350 standard deduction instead of $24,150. It skipped the zero-rate band and the QBI and child-care subtractions, then subtracted $346 of child credits that are not applied. It also left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed $132,020 at 5.3% with no zero-rate band, which understates taxable income relative to $138,270. It then subtracted a $440 child credit that the 2026 computation does not apply and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used the single-filer zero-rate band of about $4,930 instead of the head-of-household band of about $9,622. It taxed $139,270, leaving out the $1,000 child-care subtraction. It also subtracted $410 of child tax credits that are not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270, skipping the HoH zero-rate band of about $9,622 and the $1,000 Idaho child-care subtraction. It also left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It subtracted the $6,480 employer insurance premiums as an extra Idaho deduction, getting $131,790 instead of $138,270. It used a $9,960 zero band instead of about $9,622. It also subtracted $410 of child credits that are not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed all of $139,270 at 5.3% with no HoH zero-rate band and left out the $1,000 child-care subtraction. It then subtracted $410 of Idaho child credits that are not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It applied 5.3% to the full $139,270, skipping the roughly $9,622 HoH zero-rate band and the $1,000 child-care subtraction. It then subtracted $410 of child tax credits that are not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It got the correct $138,270 taxable income but used the single-filer zero band of $4,980 instead of the HoH band of about $9,622, which overstates the tax by about $245. It then subtracted $410 of child credits that are not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It applied a guessed 4% effective rate instead of computing 5.3% on taxable income of $138,270 above the roughly $9,622 HoH zero-rate band and adding the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It used the repealed 5.695% rate on all of $141,157 and explicitly refused the QBI deduction, which Idaho allows. It also skipped the child-care subtraction and the zero-rate band, and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $12,450 HoH standard deduction instead of $24,150 and applied the outdated 5.695% rate to all of $151,770. It skipped the QBI and child-care subtractions and the zero-rate band." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to the full $136,070 instead of 5.3% on taxable income of $138,270 above the roughly $9,622 HoH zero band. It also left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It subtracted the $6,480 insurance premiums to get a $157,740 AGI and used a $12,585 standard deduction instead of the $24,150 HoH deduction. It then applied the outdated 5.8% rate to all of $145,155 with no zero-rate band." -us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It applied 5.3% to roughly $135,000–$140,000 as a flat tax with no zero-rate band. Idaho exempts the first ~$9,622 of HoH taxable income, which gives $6,818.34, and then adds the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It explicitly refused the $800 QBI deduction and left out the $1,000 Idaho child-care subtraction, giving $140,070 instead of $138,270. It used a $2,500 zero band instead of about $9,622. It also subtracted $410 of child credits that are not applied and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used an unexplained $162,405 AGI instead of $164,220, a $3,500 HoH standard deduction instead of $24,150, and the repealed 5.695% rate. It then subtracted an invented $1,000-per-child Idaho credit. The errors offset each other only partly." -us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It deducted $3,000 of nonexistent $1,500 dependent deductions and skipped the $800 QBI and $1,000 child-care subtractions. It then applied a 5.305% rate to all of $137,070 with no zero-rate band and left out the $10 permanent building fund tax." -us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It used a married-filing-jointly standard deduction for a head-of-household filer. It made an arithmetic error that cut taxable income to $96,448 and applied the outdated 5.8% rate instead of 5.3% above the HoH zero band." -us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It used Idaho's repealed graduated brackets of 1%, 3% and 5.5%, an invented $2,400 Idaho standard deduction on top of federal taxable income, and an understated $121,620 federal taxable income. The 2026 law applies a flat 5.3% to taxable income of $138,270 above the roughly $9,622 zero band." -us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly picked the refundable Idaho grocery credit and counted 3 exemptions. However, it used the old $120 per-person amount instead of the $155 per-person amount for non-seniors in 2026, so it got $360 instead of $465." -us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model wrongly said Idaho has no refundable individual income tax credits. It missed the refundable grocery credit, which has no income limit and pays $155 for each of the 3 household members, for $465 total." -us,scenario_076,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model missed Idaho's refundable grocery credit. That credit applies at any income level and pays $155 each for the head and two children, so the correct total is $465, not $0." -us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model identified the grocery credit and the 3-person count but replaced the correct rate with a made-up $170 per person. The 2026 non-senior amount is $155, giving $465 rather than $510." -us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model wrongly treated Idaho's grocery credit as a nonrefundable credit that is refundable only in limited cases. The grocery credit is fully refundable regardless of income and pays $155 × 3 = $465. -us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model wrongly called the Idaho grocery credit nonrefundable or unavailable at this income. It is a refundable per-exemption credit with no income phase-out, worth $155 each for 3 people, or $465." -us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model wrongly classified Idaho's grocery credit as nonrefundable. The credit is refundable and pays $155 per exemption for the head and two children, for a total of $465." -us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged that the grocery credit is refundable but then left it out, claiming it was 'already reflected elsewhere.' The state_refundable_credits output is exactly where the grocery credit is counted: $155 × 3 = $465." -us,scenario_076,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit amount. For 2026 the non-senior amount is $155, so 3 people get $465, not $360." -us,scenario_076,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model counted Idaho's $205-per-child child tax credit as refundable, but that credit is nonrefundable. It also missed the refundable grocery credit, which is $155 × 3 people = $465." -us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated Idaho's nonrefundable $205 child tax credit as refundable and left out the refundable grocery credit. The correct refundable amount is the grocery credit of $155 for each of 3 members, or $465." -us,scenario_076,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the right credit and member count but the superseded $120 per-person rate. With the $155 non-senior amount for 2026, the total is $465, not $360." -us,scenario_076,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied the old $120 per-person grocery credit. For 2026 the amount is $155 per non-senior exemption, so 3 people get $465." -us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly treated the grocery credit as refundable for all 3 members but used the outdated $120 amount. The 2026 rate is $155 per person, for $465." -us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model assumed Idaho's refundable credits are income-limited and returned $0. The grocery credit has no income test and pays $155 for each of 3 exemptions, or $465." -us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model wrongly concluded that no Idaho refundable credit applies at this income. It missed the grocery credit, which is refundable at any income and pays $155 × 3 = $465." -us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a stale $100 per-person grocery credit amount. For 2026 the non-senior amount is $155, so the total is $465, not $300." -us,scenario_076,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model returned $0 and missed Idaho's refundable grocery credit. That credit gives $155 for each of the head and two children, for $465." -us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model said the household qualifies for no refundable Idaho credit. The grocery credit applies to every resident exemption without an income limit, giving $155 × 3 = $465." -us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model identified the grocery credit and 3 members but used the old $120 amount. The 2026 per-person amount of $155 gives $465. -us,scenario_076,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,The model wrongly claimed PolicyEngine models no refundable Idaho credits. Idaho's refundable grocery credit applies and equals $155 × 3 = $465. -us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the long-superseded $100 per-member grocery credit amount. The 2026 non-senior amount is $155, so 3 members get $465." -us,scenario_076,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model applied a $100 per-person grocery credit instead of the 2026 amount of $155, so it got $300 instead of $465." -us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable Idaho credit and missed the grocery credit. That credit is refundable for every exemption at any income level, giving $155 × 3 = $465." -us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed Idaho's refundable credits have income thresholds this household exceeds. The grocery credit has no income phase-out and pays $155 per exemption, for $465." -us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly used 3 non-senior members but the outdated $120 grocery credit rate. At the 2026 rate of $155 per person, the total is $465." -us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model said no refundable Idaho credit applies and left out the grocery credit. That credit pays $155 for each of the 3 household members, for $465." -us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the superseded $120 per-person grocery credit. The 2026 non-senior amount is $155, giving $465 for 3 members." -us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model missed Idaho's refundable grocery credit, which applies at any income level and pays $155 × 3 = $465." -us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $100 per-person grocery credit instead of the 2026 amount of $155 per non-senior exemption, so it got $300 instead of $465." -us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model wrongly treated $100 per resident as permanent law for 2026. Current law sets the non-senior grocery credit at $155, so 3 residents get $465." -us,scenario_076,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the old $120 grocery credit rate. The 2026 amount is $155 per person, giving $465 for the head and two children." -us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model wrongly assumed the Idaho grocery credit phases out at higher incomes. The credit has no income phase-out and pays $155 per exemption, or $465 for 3 members." -us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an outdated $100 per-member grocery credit. The 2026 non-senior amount is $155, so 3 members get $465." -us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there was nothing to score. The correct answer is the Idaho grocery credit of $155 × 3 = $465." -us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly found 3 non-senior exemptions but used the old $120 grocery credit. The 2026 amount of $155 per person gives $465. -us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model wrongly treated the grocery credit as not triggered at this income. The credit is refundable for every resident exemption with no income test, giving $155 × 3 = $465." -us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly left out any senior supplement but used a $100 base rate. The 2026 non-senior grocery credit is $155 per member, so the total is $465." -us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model looked only for income-tested credits such as an earned income credit. It missed Idaho's refundable grocery credit, which pays $155 for each of 3 members regardless of income, for $465." -us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model said no Idaho refundable credits apply, missing the grocery credit. That credit pays $155 per exemption for the head and two children, for $465." +us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model left out the $1,000 Idaho dependent-care subtraction, which gave $139,270 of taxable income instead of $138,270. It applied 5.3% from the first dollar instead of above the $9,622 HoH (joint-schedule) zero-rate amount, subtracted a $410 child credit that does not apply, skipped the $10 permanent building fund tax, and then nudged the result up to $7,099 for no stated reason." +us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model left out the $800 QBI deduction on the rental income, so taxable income came out $139,070 instead of $138,270. It taxed the full amount at 5.3% with no $9,622 HoH zero-rate amount, subtracted a $410 child credit that does not apply, and skipped the $10 permanent building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model counted $9,600 of non-taxable child support as income and subtracted ESI premiums from AGI. It used a $15,000 single standard deduction instead of the $24,150 HoH amount and added a made-up $9,380 adjustment. On top of that it applied graduated brackets instead of Idaho's 5.3% flat rate above the $9,622 HoH zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model gave a rough 5.3% approximation and subtracted dependent credits that do not apply in this computation. The correct steps are $138,270 of taxable income (after the standard deduction, the $1,000 dependent-care subtraction and the $800 QBI deduction), 5.3% on the amount above $9,622 ($6,818.34), plus the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used the superseded 5.695% rate instead of 2026's 5.3%, and a $23,625 standard deduction instead of $24,150. It left out the $1,000 dependent-care subtraction and the $800 QBI deduction, and it raised its own pre-credit figure to $8,204 even after describing a grocery credit reduction." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model applied the old 5.695% rate above a made-up $3,420 threshold instead of 5.3% above the $9,622 HoH zero-rate amount. It used a $22,500 standard deduction instead of $24,150 and left out the dependent-care and QBI subtractions. It then raised its own $7,876 figure to $9,112 with no basis." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model used a $24,500 standard deduction, left out the $1,000 dependent-care and $800 QBI subtractions, and applied 5.3% from the first dollar with no $9,622 HoH zero-rate amount. Its final $7,480 is higher than its own pre-credit estimate even though it said a credit reduced the tax." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model used $140,070 of taxable income, leaving out the $1,000 Idaho dependent-care subtraction and the $800 QBI deduction. It used a $9,346 zero-rate amount instead of the 2026 HoH $9,622. It then subtracted a $410 child credit that does not apply and left out the $10 permanent building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model filed as single with a $15,000 deduction instead of HoH at $24,150, and subtracted $4,714 personal exemptions that Idaho no longer allows. It applied the old 5.695% rate instead of 5.3% above the $9,622 HoH zero-rate amount and subtracted a $100 child care credit." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model used a $16,000 standard deduction instead of the $24,150 HoH amount and the old 5.695% rate instead of 5.3% above $9,622. It then raised its own $8,443 figure to $9,400 for no stated reason, and it never took the dependent-care or QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model taxed $140,070 at 5.3% from the first dollar, leaving out the $1,000 dependent-care subtraction, the $800 QBI deduction and the $9,622 HoH zero-rate amount. It subtracted a $410 child credit that does not apply and left out the $10 permanent building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model used the outdated 5.8% rate and a $19,900 standard deduction instead of 5.3% and $24,150. It also left out the dependent-care and QBI subtractions, the $9,622 HoH zero-rate amount and the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model subtracted the $6,480 ESI premiums from AGI, giving $157,740 instead of $164,220, and used a $23,200 standard deduction. It applied the outdated 5.8% rate from the first dollar instead of 5.3% above the $9,622 HoH zero-rate amount, and it left out the dependent-care and QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model subtracted ESI premiums from AGI, giving $157,740, and used a $12,155 HoH deduction instead of $24,150. It applied the old 5.695% rate instead of 5.3% above $9,622, and its submitted $8,247.54 doesn't match its own computed $8,291.07." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied a 5.0% rate instead of Idaho's 5.3% above the $9,622 HoH zero-rate amount. It left out the $1,000 dependent-care and $800 QBI subtractions, subtracted $450 of child and CDCC credits that do not apply, and skipped the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used a made-up $130,867 taxable income instead of $138,270 and the outdated 5.8% rate instead of 5.3% above $9,622. It also subtracted a $410 child credit that does not apply." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model gave no derivation. Its $8,100 matches an old-rate (5.695%–5.8%) flat tax on AGI less the standard deduction. The correct figure is 5.3% on $138,270 above the $9,622 HoH zero-rate amount, plus $10." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied the superseded 5.695% rate instead of 5.3%, and used an understated $129,750 taxable income instead of $138,270. It also never applied the $9,622 HoH zero-rate amount or added the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model used the outdated 5.8% rate instead of 2026's 5.3% applied above the $9,622 HoH zero-rate amount. It also subtracted child credits that do not apply in this computation." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave no derivation. Its $7,322 is about 5.3% of the full $138,270 taxable income ($7,328), so it left out the $9,622 HoH zero-rate amount, which cuts the tax to $6,818.34 before the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model gave no derivation. Its $6,985 matches 5.3% of roughly $139k–$140k from the first dollar minus a $410 child credit. That shortcut leaves out the $9,622 HoH zero-rate amount and the $1,800 of dependent-care and QBI subtractions, and wrongly takes a credit that does not apply here." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model gave no derivation. Its $7,665 matches the old 5.695% rate on roughly $134.6k, or 5.3% from the first dollar on an overstated base. Neither applies 5.3% only above the $9,622 HoH zero-rate amount on $138,270." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model claimed PolicyEngine does not model Idaho income tax and submitted $0, but PolicyEngine does compute Idaho tax. The correct steps are $138,270 of taxable income, 5.3% on the amount above $9,622, and the $10 building fund tax, for $6,828.34." +us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model applied the old 5.695% rate from the first dollar instead of 5.3% above the $9,622 HoH zero-rate amount. It used a $22,500 deduction instead of $24,150 and left out the $1,000 dependent-care and $800 QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model started from federal taxable income, which already reflects the standard deduction, and then subtracted a second $23,625 Idaho standard deduction, cutting taxable income to $116,170. It also subtracted a $410 child credit that does not apply and never applied the $9,622 zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model gave no derivation. Its $6,494 is below the correct figure: $138,270 of taxable income taxed at 5.3% above the $9,622 HoH zero-rate amount gives $6,818.34, plus the $10 building fund tax. That points to an extra credit or deduction that does not apply." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model mixed itemized medical assumptions into a rough estimate instead of following Idaho's steps. Those steps are the $24,150 HoH standard deduction, the $1,000 dependent-care and $800 QBI subtractions, 5.3% above $9,622, and the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model used a $23,350 deduction instead of $24,150, left out the dependent-care and QBI subtractions, and taxed from the first dollar with no $9,622 HoH zero-rate amount. It also subtracted two $173 child credits that do not apply and skipped the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied 5.3% from the first dollar to an understated $132,020 instead of to the $138,270 amount above the $9,622 HoH zero-rate amount. It also subtracted a $440 child credit that does not apply and left out the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model used the single-filer zero-rate threshold (about $4,930) instead of the joint-schedule $9,622 that applies to head-of-household filers, and it left out the $1,000 dependent-care subtraction. It also subtracted $410 of child credits that do not apply and skipped the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model applied 5.3% to all of $139,270, leaving out the $1,000 dependent-care subtraction and the $9,622 HoH zero-rate amount. It also left out the $10 permanent building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model subtracted the $6,480 ESI premiums a second time, cutting taxable income to $131,790 instead of $138,270. It used a $9,960 threshold instead of $9,622, subtracted $410 of child credits that do not apply, and left out the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model left out the $1,000 dependent-care subtraction and taxed $139,270 at 5.3% from the first dollar, skipping the $9,622 HoH zero-rate amount. It subtracted a $410 child credit that does not apply and left out the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model taxed $139,270 at 5.3% from the first dollar, leaving out the $1,000 dependent-care subtraction and the $9,622 HoH zero-rate amount. It subtracted $410 of child credits that do not apply and left out the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model got the $138,270 taxable income right but used a $4,980 single-filer zero-rate threshold instead of the joint-schedule $9,622 that head-of-household filers use. It also subtracted $410 of child credits that do not apply and left out the $10 permanent building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model applied a guessed 4% effective rate to gross income instead of following Idaho's steps. Those steps are $138,270 of taxable income, taxed at 5.3% above the $9,622 HoH zero-rate amount, plus the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model applied the old 5.695% rate from the first dollar to $141,157, which wrongly denies Idaho's conformity to the $800 QBI deduction. It also left out the $1,000 dependent-care subtraction and the $9,622 HoH zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model used a $12,450 HoH standard deduction instead of the 2026 $24,150 and applied the old 5.695% rate instead of 5.3% above $9,622. It also left out the dependent-care and QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model applied the superseded 5.695% rate from the first dollar to $136,070. The correct base is $138,270, taxed at 5.3% only above the $9,622 HoH zero-rate amount, plus the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model subtracted ESI premiums from AGI ($157,740), used a $12,585 deduction instead of $24,150, and applied the outdated 5.8% rate instead of 5.3% above $9,622. It also left out the dependent-care and QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model applied roughly 5.3% to all of a $135k–$140k base. It never took out the $9,622 HoH zero-rate amount, which lowers the tax to $6,818.34 before the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for this output, so there was no answer to grade." +us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model left out the $1,000 Idaho dependent-care subtraction and the $800 QBI deduction (taxable $140,070 instead of $138,270). It used a $2,500 zero-rate amount instead of the HoH $9,622, subtracted $410 of child credits that do not apply, and left out the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model used a wrong $162,405 AGI, a $3,500 HoH standard deduction instead of $24,150, and the old 5.695% rate. It then subtracted a made-up $2,000 child credit to land on its figure." +us,scenario_076,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"The model applied about 5.69% instead of 5.3% above $9,622 to $140,070, leaving out the dependent-care and QBI subtractions. It subtracted $410 of child credits that do not apply and a $465 grocery credit that does not reduce this before-refundable-credits figure." +us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model subtracted made-up $1,500-per-dependent deductions instead of the actual $1,000 dependent-care subtraction and $800 QBI deduction. It applied a 5.305% rate from the first dollar instead of 5.3% above the $9,622 HoH zero-rate amount, and it left out the $10 building fund tax." +us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the filer as married filing jointly with a $30,000 deduction and then subtracted down to an arithmetically wrong $96,448 taxable income. It also applied the outdated 5.8% rate instead of 5.3% above $9,622 on $138,270." +us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model applied repealed graduated brackets (1%/3%/5.5%) instead of the 5.3% flat rate above $9,622. It started from an understated $121,620 federal taxable income and then subtracted a made-up $2,400 Idaho standard deduction on top of it." +us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly picked the refundable grocery credit and counted 3 members. However, it used the outdated $120 per-person amount instead of the $155 that applies in 2026, so it got $360 instead of $465." +us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said Idaho has no refundable state credits and missed the refundable grocery credit. That credit pays $155 per household member with no income test, which comes to $465 here." +us,scenario_076,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which is $155 for each of the 3 household members regardless of income, so it gave $0 instead of $465." +us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly applied the grocery credit to 3 people. However, it dropped its $120 figure and replaced it with an invented $170 per-person amount instead of the $155 that applies in 2026, giving $510 instead of $465." +us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated Idaho's grocery credit as nonrefundable at this income level. In fact it is fully refundable with no income phase-out, paying $155 per member, so the household gets $465." +us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model said the grocery credit is nonrefundable or does not apply at this income. Idaho's grocery credit is refundable with no income limit and pays $155 for each of the 3 members, so the correct total is $465." +us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model wrongly treated the Idaho grocery credit as nonrefundable. It is refundable at $155 per household member, which gives $465 for this household of 3." +us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model admitted the grocery credit is refundable but still left it out, claiming it was 'already reflected elsewhere'. The grocery credit is exactly what this output is meant to capture: $155 × 3 = $465." +us,scenario_076,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model counted Idaho's $205 child tax credit as refundable, but that credit is nonrefundable and does not count here. It also left out the refundable grocery credit, which is $155 × 3 members = $465." +us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model wrongly labeled Idaho's $205-per-child credit as refundable when it is nonrefundable. It also skipped the refundable grocery credit, which is $155 for each of the 3 members ($465)." +us,scenario_076,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly picked the refundable grocery credit for 3 members. However, it used the outdated $120 per-person amount instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit instead of the $155 that applies in 2026, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the grocery credit to all 3 members, but at the outdated $120 per person instead of $155, so it got $360 instead of $465." +us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model assumed income ruled out every Idaho refundable credit. The grocery credit has no income phase-out and pays $155 per member, so the household gets $465." +us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model said no Idaho refundable credit applies at this income. The refundable grocery credit is available at any income level and is $155 × 3 = $465. +us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used the long-outdated $100 per-person grocery credit instead of $155, giving $300 instead of $465." +us,scenario_076,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 for each of the 3 household members, so it gave $0 instead of $465." +us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model concluded the household qualifies for no refundable Idaho credit and missed the grocery credit. That credit has no income limit and pays $155 per member, totaling $465." +us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model claimed PolicyEngine models no refundable credit for Idaho. PolicyEngine includes Idaho's refundable grocery credit at $155 per member, which gives $465 here." +us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the outdated $100 per-person grocery credit instead of the $155 that applies in 2026, giving $300 instead of $465." +us,scenario_076,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit for 3 members but used $100 per person instead of $155, giving $300 instead of $465." +us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit. Every resident member qualifies at $155, so the total is $465." +us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model assumed income thresholds rule out Idaho refundable credits. The grocery credit has no income limit and pays $155 × 3 = $465. +us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly applied the grocery credit to 3 members under 65 but used the outdated $120 per-person amount instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model concluded no refundable Idaho credit applies and missed the grocery credit, which pays $155 for each of the 3 members ($465)." +us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 per household member with no income test, so it gave $0 instead of $465." +us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an outdated $100 per-person grocery credit instead of $155, giving $300 instead of $465." +us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model claimed $100 per resident is the permanent-law amount for 2026. In fact the grocery credit is $155 per non-senior member, so 3 members get $465, not $300." +us,scenario_076,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used the outdated $120 per-person grocery credit instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model assumed the grocery credit phases out at this income level. Idaho's grocery credit has no income phase-out and pays $155 for each of the 3 members, so the total is $465." +us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an outdated $100 per-person grocery credit instead of $155, giving $300 instead of $465." +us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model gave no value for state_refundable_credits. The correct value is Idaho's grocery credit of $155 × 3 members = $465. +us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly applied the grocery credit to 3 members under 65 but used the outdated $120 per-person amount instead of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the grocery credit as not triggered at this income level. It applies automatically at $155 per resident household member regardless of income, totaling $465." +us,scenario_076,state_refundable_credits,muse-wonder,llm_error,categorical_eligibility,False,The model treated all Idaho credits as nonrefundable at this income. The grocery credit is refundable and pays $155 × 3 = $465. +us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit for 3 members under 65 but used $100 per person instead of $155, giving $300 instead of $465." +us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model looked only for an earned income credit and missed Idaho's refundable grocery credit, which pays $155 per household member, or $465 for this household." +us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model left out Idaho's refundable grocery credit, which pays $155 for each of the 3 household members, so it gave $0 instead of $465." us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own reasoning reached the correct result: AGI $23,006, the $16,100 standard deduction, taxable income $6,906, and $691 of tax at 10%. It then submitted $951, which matches nothing in its derivation." us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 OBBBA figure of $16,100. It also subtracted the $8,389 ESI premiums from wages that were stated as gross, and then submitted $1,402, a number that matches none of its intermediate calculations." us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It left out the $3,000 capital-loss deduction, so its taxable income was about $9,906 instead of $6,906. It then estimated about $1,000 of tax and still submitted $0, even though no nonrefundable credit applies." @@ -6317,105 +6392,107 @@ us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals us,scenario_077,snap,gemini-3.7-flash,llm_error,other,False,"The model’s explanation correctly states that the one-person household exceeds the SNAP net-income limit and explicitly concludes “value = 0,” but it submitted 276. The numeric output contradicts its own eligibility conclusion and should have been 0." us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income without applying the one-person SNAP net-income and benefit calculations to its $26,006 of wages. After the allowable earned-income and standard deductions, the household does not generate an eligible allotment, so the asserted $2,400 benefit has no valid computation." us,scenario_077,snap,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly awarded the $23 minimum allotment after determining that the regular formula produced zero. Broad-based categorical eligibility does not entitle a household failing the applicable net-income or benefit test to the minimum allotment; the minimum applies only to an otherwise eligible one- or two-person household. -us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction instead of the 2026 inflation-adjusted $12,835 and got $315.18. It then replaced that with $496, citing a 'deduction structure' it never identified. The correct figure is 3% of ($23,006 − $12,835), which is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"The model applied the 2025 single standard deduction of $12,500. Louisiana indexes that deduction for inflation from 2026, so it is $12,835. That leaves $10,171 taxable, not $10,506, and tax of $305.14 rather than $315.18." -us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"The model said Louisiana has no state income tax. Louisiana taxes individual income at a flat 3%, and here that applies to $10,171 of taxable income after the $12,835 standard deduction, giving $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"The model assumed exemptions and deductions brought the taxable base to zero. Louisiana's only deduction for a single filer is the $12,835 indexed standard deduction, which leaves $10,171 of the $23,006 AGI taxable at 3%, or $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"The model used the flat 3% rate correctly but took the $12,500 standard deduction without the 2026 inflation indexing that raises it to $12,835. That overstated taxable income by $335, giving $315 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 from $23,006 AGI. The 2026 Louisiana single standard deduction is the inflation-indexed $12,835, so the correct tax is 3% × $10,171 = $305.14, not $315." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"The model first reached about $315 using the unindexed $12,500 deduction. It then raised the answer to $446, citing 'capital-loss treatment and rounding', which has no basis. Federal AGI already reflects the $3,000 loss cap, and with the indexed $12,835 deduction the tax is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the 2025 $12,500 standard deduction rather than the 2026 inflation-adjusted $12,835. That gave $10,506 of taxable income and $315.18 of tax, instead of $10,171 and $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"The model combined the new 3% rate with the old Louisiana structure: a $4,500 personal exemption and a deduction for federal income tax paid. The 2025 reform repealed both and replaced them with a single standard deduction, $12,835 in 2026. Its taxable income of $17,705 should have been $10,171." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"The model computed about $315 with the unindexed $12,500 deduction, then added unexplained 'federal-based deductions' and 'personal exemption credits' to reach $448. Louisiana has no such add-ons after the reform. With the indexed $12,835 deduction the tax is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"The model assumed a $12,500 standard deduction and missed the 2026 inflation indexing to $12,835. Its tax came out at $315.18 instead of 3% × $10,171.18 = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"The model's $7,606 taxable base means it subtracted a $15,400 deduction from $23,006 AGI. That is close to a federal standard deduction, not Louisiana's. The Louisiana single standard deduction for 2026 is $12,835, which leaves $10,171 taxable and $305.14 of tax." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction. Louisiana's 2026 single standard deduction is $12,835 after inflation adjustment, so taxable income is $10,171 and tax is $305.14, not $315.18." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"The model applied repealed pre-2025 Louisiana law: a deduction for federal income tax paid, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. For 2026 the law is a flat 3% on AGI minus the $12,835 indexed standard deduction, which gives $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 instead of the 2026 inflation-indexed Louisiana standard deduction of $12,835. That overstated taxable income by $335 and gave $315.18 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $8,389 ESI premiums from wages a second time, reaching a $14,617 AGI; the correct AGI is $23,006. It also used the repealed $4,500 exemption and the old 1.85% bottom bracket. Under 2026 law, 3% × ($23,006 − $12,835) = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation. The correct calculation is $23,006 AGI minus the $12,835 indexed standard deduction, giving $10,171 taxed at 3%, or $305.14. Its $158 implies only about $5,300 taxable at 3%, so it overstated deductions or exemptions under the old pre-reform structure." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the ESI premiums from wages again, reaching a $14,617 AGI instead of $23,006. It stacked a $4,500 standard deduction and a $4,500 personal exemption, both repealed, and used the old 1.85% bracket. Under 2026 law the tax is a flat 3% on $23,006 − $12,835, which is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"The model cut AGI to $14,617 by subtracting ESI premiums that are not deducted from the reported gross wages, so AGI should be $23,006. It then applied the repealed $4,500 exemption and the 1.85% bracket instead of the $12,835 standard deduction and the flat 3% rate, which give $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions and exemptions eliminate the tax. The only deduction is the $12,835 single standard deduction, which leaves $10,171 of the $23,006 AGI taxable at 3%, or $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"The model's $315.18 matches 3% of $23,006 minus the 2025 deduction of $12,500. It missed that Louisiana indexed the deduction to $12,835 for 2026, which makes the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"The model relied on a pre-reform 'standard deduction and personal exemption' structure, and its explanation states $130 while it submitted $152. Both figures fall short of the correct 3% × ($23,006 − $12,835) = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"The model asserted that PolicyEngine does not model Louisiana income tax and answered $0. Louisiana's flat 3% tax applies to the $10,171 left after the $12,835 standard deduction, giving $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"The model used a $12,500 single standard deduction rather than the 2026 inflation-adjusted $12,835. Taxable income should be $10,171, not $10,506, and tax $305.14, not $315.18." -us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"The model taxed $10,506 at the repealed 1.85% bottom bracket instead of Louisiana's flat 3%. It also used the unindexed $12,500 deduction rather than $12,835. The correct tax is 3% × $10,171 = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"The model claimed taxable income after the deduction and exemption falls below a bracket threshold. Louisiana has no zero bracket and no separate exemption. Taxable income is $23,006 − $12,835 = $10,171, taxed at 3% for $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"The model assumed low wages produce no tax and skipped the calculation. $23,006 AGI minus the $12,835 standard deduction leaves $10,171 taxable at 3%, which is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the 2025 $12,500 single standard deduction and missed its 2026 inflation indexing to $12,835. Its tax was $315.18 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"The model handled the $3,000 capital-loss cap correctly but used the unindexed $12,500 deduction instead of the 2026 amount of $12,835. That overstated taxable income by $335 and tax by $10.04." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 rather than the inflation-indexed 2026 Louisiana standard deduction of $12,835. That gave $10,506 taxable and $315.18 of tax instead of $10,171.18 and $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"The model stacked a $4,500 personal exemption on top of the standard deduction. The 2025 reform repealed the personal exemption and folded it into the standard deduction, which is $12,835 for 2026. Taxable income is $10,171, not $6,006, so tax is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"The model correctly indexed the $12,500 deduction for 2026 but overshot the adjustment, using $12,875 instead of $12,835. That understated taxable income by $40 and gave $303.93 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction. Louisiana's 2026 single standard deduction is $12,835, so the tax is 3% × $10,171.18 = $305.14, not $315.18." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"The model applied the 2025 $12,500 single-filer deduction and missed the 2026 inflation indexing to $12,835. Its result was $315.18 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction rather than the 2026 indexed $12,835. That overstated taxable income by $335 and gave $315.18 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions and exemptions bring taxable income to zero. The $12,835 Louisiana standard deduction is well below the $23,006 AGI, which leaves $10,171 taxable at 3%, or $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"The model applied repealed pre-2025 law: a $4,500 exemption and graduated 1.85%/3.5% brackets. The 2026 law is a $12,835 standard deduction and a flat 3% rate, which gives $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"The model used the flat 3% rate but the unindexed $12,500 deduction. With the 2026 inflation-adjusted $12,835 deduction, taxable income is $10,171 and tax is $305.14, not $315." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"The model correctly noted that the federal-tax deduction was repealed but used the 2025 $12,500 standard deduction instead of the 2026 indexed $12,835. Its tax was $315.18 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $8,389 ESI premiums from the reported gross wages, giving a $14,617 AGI instead of $23,006. It also used the repealed $4,500 deduction and the old 1.85% bracket. The correct tax is 3% × ($23,006 − $12,835) = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"The model used a $12,500 single standard deduction for 2026. Louisiana indexes that deduction to $12,835 for 2026, which makes taxable income $10,171 and tax $305.14 rather than $315." -us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $305.14 reference." -us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"The model applied the unindexed $12,500 deduction rather than the 2026 inflation-adjusted $12,835. That gave $315.18 instead of 3% × $10,171.18 = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"The model said Louisiana has no state income tax. Louisiana levies a flat 3% individual income tax, which here applies to $23,006 AGI minus the $12,835 standard deduction, giving $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the federal standard deduction of $16,100 plus a repealed $4,500 Louisiana exemption. Louisiana allows only its own $12,835 single standard deduction for 2026. Taxable income is $10,171, not $2,406, and tax is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"The model did not cap the net capital loss at $3,000, so its AGI was $18,038 instead of $23,006. It then used the repealed $4,500 exemption and graduated 1.85%/3.5% brackets instead of the $12,835 standard deduction and the flat 3% rate, which give $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation. At 3%, its $347.24 implies about $11,575 of taxable income, which means it subtracted roughly $11,431 from the $23,006 AGI. The correct deduction is the 2026 indexed $12,835, which leaves $10,171 taxable and $305.14 of tax." +us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"The model computed $315.18 using the $12,500 deduction, then raised it to $496 based on an unspecified 'deduction structure' adjustment. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and flat 3% rate to obtain $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"The model claimed Louisiana has no state individual income tax, although Louisiana levies a flat 3% tax. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"The model asserted that a large personal exemption plus the standard deduction wipes out the taxable base. For this household, the frozen computation applies no separate exemption and leaves $10,171.18 taxable at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"The model applied the flat 3% rate to AGI minus the $12,500 deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 from $23,006 and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"The model reached about $315 using the $12,500 deduction, then raised it to $446, citing capital-loss treatment. The $3,000 capital-loss cap is already reflected in its $23,006 AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"The model subtracted a $4,500 personal exemption and an $801 federal-income-tax deduction from $23,006 AGI, leaving $17,705 taxable at 3% and about $531 of tax. Louisiana ended the federal income tax deduction after 2021; the 2025 reform replaced the personal exemption with a standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"The model computed about $315 using the $12,500 deduction, then added unspecified 'federal-based adjustments' to reach $448. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"A taxable base of $7,606 means the model subtracted $15,400 from its $23,006 AGI, calling it the standard deduction without stating the amount. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"The model used a federal-income-tax deduction that Louisiana ended after 2021, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $8,389 ESI premiums from wages, giving $14,617 AGI rather than the reference's $23,006.18, then used a repealed $4,500 exemption and 1.85% bracket. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation for $158, which is below the frozen reference and consistent with outdated rates or excess deductions. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"The model used $14,617 AGI after subtracting ESI premiums, then stacked a $4,500 standard deduction and $4,500 personal exemption and applied the repealed 1.85% bracket. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"The model used $14,617 AGI after subtracting ESI premiums and applied a repealed $4,500 exemption and 1.85% rate. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions and exemptions eliminate the tax. The frozen computation leaves $10,171.18 taxable at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"The $315.18 answer reflects $23,006 AGI minus the $12,500 deduction at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"The model submitted $152 while its explanation stated $130, with no derivation for either figure. Both are below the frozen reference. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"The model wrongly asserted that PolicyEngine does not model Louisiana income tax. PolicyEngine computes the flat 3% tax on AGI minus the $12,835 standard deduction, which is $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"The model applied the repealed 1.85% rate to AGI minus the $12,500 deduction. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"The model claimed taxable income falls below the lowest-bracket threshold. Louisiana has a flat 3% rate with no zero bracket, and the frozen computation leaves $10,171.18 taxable. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"The model assumed low wage income produces zero tax without computing it. AGI of $23,006.18 less the $12,835 standard deduction leaves $10,171.18 taxed at 3%, which is $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"The model stacked a $4,500 personal exemption on top of a $12,500 deduction. The 2025 reform folded the exemption into a standard deduction; the frozen computation leaves $10,171.18 taxable at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"The model correctly recognized that the deduction is inflation-indexed but used $12,875 instead of PolicyEngine's 2026 value of $12,835. The $12,875 is the Department's provisional withholding and estimated-tax amount; its $40 difference from the frozen deduction explains $1.20 of the $1.21 tax gap, while the prompt's omitted $0.18 of wages and cent rounding explain the remainder. This output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"The model claimed taxable income is zero or negative. AGI of $23,006.18 exceeds the $12,835 standard deduction, leaving $10,171.18 taxed at 3% for $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"The model used the repealed $4,500 exemption and 1.85%/3.5% graduated brackets. Louisiana's 2025 reform replaced these with a flat 3% rate and a standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"The model got the flat 3% rate and federal-tax-deduction repeal right but used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"The model subtracted ESI premiums to obtain $14,617 AGI, then applied a repealed $4,500 deduction and 1.85% bracket. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for this output, so no answer could be parsed. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"The model falsely stated that Louisiana has no state income tax; Louisiana imposes a flat 3% tax. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,muse-wonder,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $16,100 federal standard deduction plus a repealed $4,500 personal exemption. The frozen Louisiana computation applies its own standard deduction and no separate exemption. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"The model netted the full $7,968 capital loss against wages instead of applying the $3,000 cap, giving $18,038 AGI, then applied a repealed $4,500 exemption and 1.85%/3.5% brackets. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." +us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation; $347.24 at 3% implies about $11,575 taxable income and a deduction of roughly $11,431 from $23,006 AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994." us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former 5% refundable EITC in tax year 2026. That state credit is unavailable for 2026, so multiplying an estimated federal EITC by 5% incorrectly produced $10.53 instead of zero." us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former refundable EITC rate of 5% to a purported $108 federal EITC. Louisiana provides no refundable EITC for tax year 2026, so the state refundable credit is zero." us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly carried Louisiana’s former 5% refundable EITC into 2026 and also assigned a $625 federal EITC to a childless filer with $26,006 of earnings, above the childless-credit phaseout range. The unavailable state credit cannot produce the claimed $31.25." us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract." us,scenario_077,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $26,460 of real estate taxes as SALT and left out the Maryland state and county income tax, which pushes SALT to the $40,400 cap. That understated itemized deductions at $54,670 instead of $68,610. It also made an arithmetic error: its own bracket sum was $27,510, but it reported about $33,499." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Put Maryland income tax at about $6,905, leaving out the county local income tax and understating the state tax, so SALT came to $33,365 instead of the $40,400 cap. Taxable income came out about $7,000 too high ($138,562 instead of $131,527)." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly subtracted $2,606 of unreimbursed employee business expenses above the line, even though they are not deductible. It also left Maryland state and local income tax out of SALT, so SALT was $26,460 instead of the $40,400 cap. Its bracket math does not match even its own taxable income of $142,861." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Never computed the tax and entered $0, even though its own reasoning said taxable income was well above $100k. The correct calculation (itemized deductions of $68,610, taxable income of $131,526.91, 2026 brackets) gives $24,164.46." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the pre-OBBBA $10,000 SALT cap. For 2026 the cap is $40,400, and real estate taxes plus Maryland income tax reach that cap. As a result, itemized deductions were understated by $30,400 and taxable income was overstated at $161,927." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at the old $10,000 instead of the 2026 $40,400 cap that real estate taxes plus Maryland income tax reach. It then abandoned its own $31,424 calculation and reported an unsupported $36,757." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Used the expired $10,000 SALT cap instead of the 2026 OBBBA cap of $40,400. That cut itemized deductions to $38,210 instead of $68,610 and left taxable income about $30,400 too high." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly used the $40,400 cap but put Maryland income tax at only $6,936, leaving out the county local income tax, so SALT was $33,396 instead of the capped $40,400. Taxable income came out $138,531 instead of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the $10,000 SALT cap instead of the 2026 $40,400 cap, which overstated taxable income at $161,927. It also used made-up inflation-adjusted brackets instead of the actual 2026 thresholds." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the old $10,000 SALT cap instead of $40,400 and left the $1,632 refund out of AGI. It then raised its own $31,069 result to $42,800 by wrongly folding in Medicare surtax, which is not part of income tax before credits." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Estimated Maryland income tax at about $7,000, leaving out the county local income tax, so SALT of about $33,460 fell short of the $40,400 cap that applies. Taxable income came out about $138,445 instead of $131,527." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400. That understated itemized deductions at $38,210 and overstated taxable income at $161,927." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired in 2026 and applied a personal exemption and the pre-TCJA 10/15/25/28% brackets. OBBBA made the TCJA brackets permanent, with no personal exemption and a $40,400 SALT cap." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset law: a $5,300 personal exemption, uncapped SALT, and pre-TCJA brackets. Under OBBBA, 2026 has no personal exemption, SALT is capped at $40,400, and the 10/12/22/24% brackets apply." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only real estate taxes as SALT and left out Maryland state and local income tax, which pushes SALT to the $40,400 cap. Taxable income came out $145,467 instead of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired: it used uncapped SALT, a $5,150 personal exemption, and pre-TCJA 15/25/28% brackets. OBBBA keeps the TCJA brackets with no exemption and a $40,400 SALT cap." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction instead of itemizing $68,610 (SALT capped at $40,400 plus $28,210 of mortgage interest). It also left the $1,632 refund out of income, which overstated taxable income at $183,505." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired, which brings back personal exemptions and uncapped SALT, and then added AMT. Under OBBBA the TCJA brackets continue, SALT is capped at $40,400, and no AMT applies because tentative minimum tax is below the regular tax." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: a $5,150 personal exemption and pre-TCJA brackets, with only about $6,886 of Maryland income tax counted. Current 2026 law has no exemption, the 10/12/22/24% brackets, and SALT capped at $40,400." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Used the standard deduction instead of itemizing $68,610 of SALT and mortgage interest. That roughly doubled the taxable income attributable to deductions and inflated the tax to $33,020." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Treated 2026 as post-TCJA-sunset, with uncapped SALT of $37,999, a $5,050 personal exemption, and pre-TCJA rates. Under OBBBA, SALT is capped at $40,400, there is no exemption, and the TCJA brackets apply." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $31,316 answer implies taxable income of about $161k, which matches capping SALT at the old $10,000. The correct figure uses the 2026 $40,400 cap, for itemized deductions of $68,610 and taxable income of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-expiration rules (personal exemption and pre-TCJA brackets) instead of OBBBA's permanent TCJA brackets. It also left Maryland income tax out of SALT, which should be capped at $40,400." -us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction instead of itemizing $68,610. It also subtracted a made-up $12,000 child and dependent care credit for a single filer with no children and no care expenses." -us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 $40,400 cap. That understated itemized deductions at $38,210 instead of $68,610 and overstated taxable income at $161,927." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the outdated $10,000 SALT cap and mixed up the standard deduction with itemizing. Its $42,034 even exceeds the roughly $36,767 tax under the standard deduction, so it also misapplied the brackets." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only mortgage interest and real estate taxes, leaving out Maryland state and local income tax, which raises SALT to the $40,400 cap. Taxable income was overstated by about $14,000." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and local income tax out of SALT, so it used $26,460 instead of the $40,400 cap and got taxable income of $145,467. It then misapplied the brackets: $29,576.96 does not match the $27,510.08 that its own taxable income produces." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $25,947.40 implies taxable income of about $138,956, which means it counted only part of the Maryland income tax (no county local tax) in SALT. The full state-plus-county tax plus real estate taxes reach the $40,400 cap." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to the $26,460 of real estate taxes and left out the Maryland state and county income tax, which is deductible and raises SALT to the $40,400 cap. Taxable income came out $145,467 instead of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only mortgage interest and real estate taxes and left Maryland state and local income tax out of SALT. SALT should be capped at $40,400, so it understated deductions by $13,940." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly concluded that no deductible tax payments existed besides real estate taxes. The Maryland state and county income tax on $198,505 of wages is deductible as SALT and raises SALT to the $40,400 cap." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and local income tax out of SALT and deducted only $26,460 of real estate taxes instead of the $40,400 cap. Taxable income came out $145,467 instead of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only about $6,936 of Maryland income tax and left out the county local income tax, so SALT of $33,396 stayed under the $40,400 cap instead of reaching it. Taxable income was overstated by about $7,000." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Underestimated Maryland income tax by leaving out the county local income tax, so SALT stopped at about $33,400 instead of the $40,400 cap. Taxable income came out about $138,530 instead of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no reasoning. Its $41,500 is above even the roughly $36,767 tax under the standard deduction, so it skipped the $68,610 of itemized deductions and also overstated the bracket tax." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied post-TCJA-sunset law, with a $5,427 personal exemption and pre-TCJA brackets indexed from 2017. It also left Maryland income tax out of SALT. Current law keeps the TCJA brackets with no exemption and a $40,400 SALT cap." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired and applied a $5,300 personal exemption and the 10/15/25/28% brackets. OBBBA made the TCJA rates permanent with no exemption, and SALT is capped at $40,400." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and county income tax out of SALT, so it deducted $26,460 instead of the $40,400 cap. It also used wrong lower-bracket thresholds, giving $1,220 and $4,485 instead of $1,240 and $4,560." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Used TCJA-sunset law, with uncapped SALT of $36,346, a $5,434 personal exemption, and pre-TCJA brackets. Under OBBBA, 2026 has no exemption, SALT is capped at $40,400, and the 10/12/22/24% brackets apply." -us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only real estate taxes in SALT and left out the Maryland state and local income tax, which pushes SALT to the $40,400 cap. Taxable income came out about $145,467 instead of $131,526.91." -us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no usable value for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly identified the $40,400 SALT cap but put only real estate taxes under it, leaving out the Maryland state and county income tax that brings SALT up to the cap. Itemized deductions came out $54,670 instead of $68,610." -us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Entered $0 without any calculation. A single filer with $200,137 of AGI and $68,610 of itemized deductions has $131,526.91 of taxable income and owes $24,164.46." -us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Left Maryland state and local income tax out of SALT, so SALT stopped at $26,460 instead of the $40,400 cap. That understated itemized deductions by $13,940." -us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the old $10,000 SALT cap instead of $40,400, and in the end used 2024 bracket thresholds instead of the 2026 indexed brackets." -us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Invented an AMT liability. Tentative minimum tax here is about 26% × ($171,927 AMTI − $90,100 exemption) ≈ $21,275, which is below the regular tax, so no AMT applies. It also mixed the standard deduction with itemized deductions and left Maryland income tax out of SALT." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income taxes, which push SALT to the $40,400 cap. That produced a $145,467 taxable income instead of $131,526.91. It then mis-added its own bracket sum ($27,510) and reported $33,499." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It added Maryland state income tax (~$6,905) to SALT but left out Maryland county (local) income tax. That put SALT at $33,365, below the cap, when the full amount exceeds $40,400 and is capped there. As a result, itemized deductions came out at $61,575 instead of $68,610." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted $2,606 of unreimbursed employee business expenses above the line, which is not allowed. It also limited SALT to the real estate tax, leaving out Maryland state and local income taxes that bring SALT to the $40,400 cap. Its final $25,779 does not even match the tax on its own $142,861 taxable income." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It never calculated the tax and submitted 0 as a placeholder. The correct calculation (taxable income $131,526.91 after $68,610 of itemized deductions) gives $24,164.46." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the old $10,000 TCJA SALT cap instead of the 2026 OBBBA cap of $40,400. That understated itemized deductions by $30,400 (to $38,210) and taxed $161,927 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the obsolete $10,000 instead of $40,400, which led it to $31,424. It then dropped even that figure for an unsupported $36,757, which is close to the tax under the standard deduction rather than under the correct $68,610 of itemized deductions." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap instead of the 2026 cap of $40,400, which gave $38,210 of itemized deductions instead of $68,610 and $161,927 of taxable income instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted Maryland state income tax ($6,936) in SALT but left out Maryland county local income tax. With that included, SALT exceeds the $40,400 cap, so the deduction is $40,400, not $33,396, and taxable income is $131,526.91, not $138,531." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA limit of $40,400. It also used invented brackets instead of the published 2026 single brackets (12,400/50,400/105,700), and it taxed $161,927 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap instead of $40,400 and left the $1,632 refund out of income. It then raised its own $31,069 figure to $42,800 by adding Medicare surtax effects, which do not belong in income tax before refundable credits." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It included only about $7,000 of Maryland state income tax in SALT and left out county local income tax. The full SALT amount exceeds the $40,400 cap, so itemized deductions are $68,610, not about $61,700." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the 2026 cap of $40,400. That gave $38,210 of itemized deductions and $161,927 of taxable income instead of $68,610 and $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired for 2026, adding a $5,063 personal exemption and using pre-TCJA 10/15/25/28% brackets. In fact OBBBA made the TCJA rates permanent, eliminated the exemption, and capped SALT at $40,400, so its uncapped $36,170 of SALT is also wrong." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $5,300 personal exemption and pre-TCJA brackets, which OBBBA permanently replaced. It also counted only Maryland state income tax in SALT and left out county tax. The correct calculation caps SALT at $40,400 and applies the 10/12/22/24% brackets." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions included only mortgage interest and real estate tax and left out Maryland state and county income taxes, which raise SALT to the $40,400 cap. As a result it taxed $145,467 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as a TCJA-sunset year, with uncapped SALT, a $5,150 personal exemption, and 10/15/25/28% brackets, even though OBBBA made the TCJA rates permanent and capped SALT at $40,400. It also left the $1,632 taxable refund out of income." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing, even though $40,400 of capped SALT plus $28,210 of mortgage interest ($68,610) far exceeds the $16,100 standard deduction. It also left out the $1,632 taxable refund." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired, bringing back personal exemptions and uncapped SALT and triggering AMT. Under OBBBA, SALT is capped at $40,400 and the AMT exemption is $90,100. The resulting tentative minimum tax (~$21,275) is below the regular tax, so no AMT applies." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,150 personal exemption and pre-2018 brackets, instead of the permanent OBBBA 10/12/22/24% schedule. It also counted only about $6,886 of state income tax in SALT and left out Maryland county tax, which brings SALT to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction instead of itemizing $68,610 ($40,400 of capped SALT plus $28,210 of mortgage interest). It also left out the $1,632 taxable refund, so taxable income was far too high." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had sunset, adding a $5,050 personal exemption, treating SALT as uncapped, and using pre-TCJA brackets. OBBBA kept the 10/12/22/24% structure, eliminated the exemption, and caps SALT at $40,400." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $31,316 is consistent with about $38,210 of itemized deductions, which reflects the obsolete $10,000 SALT cap. The correct itemized total is $68,610 ($40,400 of SALT plus $28,210 of interest), giving taxable income of $131,526.91 and tax of $24,164.46." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'TCJA expiration rules', meaning pre-2018 brackets and personal exemptions, even though OBBBA made the TCJA rates permanent. The correct calculation uses the 10/12/22/24% brackets on $131,526.91 after a $40,400 SALT deduction." +us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing $68,610. It also subtracted a made-up $12,000 child and dependent care credit for a single 19-year-old with no children and no care expenses." +us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the 2026 OBBBA cap of $40,400, so it itemized only $38,210 and taxed $161,927 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap and mixed up the standard and itemized deductions. Its $42,034 is higher than even the $36,766.86 tax under the standard deduction, which shows a bracket error on top of its failure to deduct $68,610 of itemized deductions." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions included only mortgage interest and real estate tax and left out Maryland state and county income taxes, which bring SALT to the $40,400 cap. Its $28,347 also does not match the bracket tax on its own $145,467 base." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and local income taxes, which push SALT to the $40,400 cap, so it taxed $145,467 instead of $131,526.91. It also miscalculated the bracket tax on its own base." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $25,947.40 corresponds to about $61,200 of itemized deductions. That means it added only Maryland state income tax to the real estate tax and left out county local income tax, which brings SALT to the full $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax ($54,670) and left out the Maryland state and county income taxes paid on $198,505 of wages. Those taxes are deductible and raise SALT to the $40,400 cap, for $68,610 in total itemized deductions." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted only $26,460 of real estate tax as SALT and left out Maryland state and county income taxes. With those included, SALT reaches the $40,400 cap and taxable income falls to $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It reasoned that 'no other deductible tax payments are reported' and left out Maryland state and county income taxes, which the engine computes and includes in SALT. With them, SALT reaches the $40,400 cap and itemized deductions total $68,610, not $54,670." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It limited SALT to the $26,460 real estate tax and left out Maryland state and local income taxes, which bring SALT to the $40,400 cap. As a result it taxed $145,467 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It added $6,936 of Maryland state income tax to SALT but left out Maryland county local income tax. The combined total exceeds the $40,400 cap, so SALT is $40,400, not $33,396." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It included only Maryland state income tax in SALT and left out county local income tax. With both included, SALT is capped at $40,400 and itemized deductions are $68,610, giving taxable income of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. Its $41,500 is higher than even the $36,766.86 tax under the standard deduction, so it reflects neither itemizing the $68,610 nor the 2026 brackets correctly." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law, with a $5,427 personal exemption and inflated pre-2018 brackets, instead of the permanent OBBBA schedule. It also left Maryland state and county income taxes out of SALT, which should be capped at $40,400." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It brought back the 10/15/25/28% brackets and a $5,300 personal exemption, both replaced by OBBBA. It also counted only $6,936 of Maryland state tax and left out county income tax, which takes SALT to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It said the SALT cap does not bind, but it counted only the real estate tax and left out Maryland state and county income taxes. With those included, SALT exceeds $40,400 and is capped there. It also used incorrect lower-bracket thresholds." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA brackets and a $5,434 personal exemption with uncapped SALT. Under OBBBA, the 2026 law keeps the 10/12/22/24% brackets, has no exemption, and caps SALT at $40,400." +us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It limited SALT to the $26,460 real estate tax and left out Maryland state and county income taxes, which push SALT to the $40,400 cap. As a result it taxed $145,467 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It submitted no value for federal_income_tax_before_refundable_credits. +us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly applied the $40,400 cap but counted only the $26,460 real estate tax as SALT. It left out Maryland state and county income taxes, which push SALT up to the cap, and so it understated itemized deductions by $13,940." +us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It reported 0 with no calculation, even though AGI is $200,136.91. After $68,610 of itemized deductions, the 2026 single brackets give $24,164.46." +us,scenario_078,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT deduction included only the real estate tax and left out Maryland state and local income taxes, which bring SALT to the $40,400 cap. That understated itemized deductions by $13,940." +us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $26,460 real estate tax as the whole SALT amount and left out Maryland state and county income taxes. With those included, SALT exceeds the cap and is limited to $40,400, which lowers taxable income to $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA limit of $40,400, and it then used 2024 brackets instead of the published 2026 single brackets." +us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It made up an AMT liability using a nonexistent 'foreign-tax-equivalent offset' and no AMT exemption. With the 2026 $90,100 AMT exemption, tentative minimum tax (~$21,275 on ~$171,927 of AMTI) is below the $24,164.46 regular tax. It also left Maryland income taxes out of SALT and misapplied the brackets." us,scenario_078,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly treated Maryland county income tax as part of the requested local_income_tax variable and applied an assumed 3.03% average county rate to taxable income. This output excludes Maryland county tax and only captures the specifically listed modeled city taxes, none of which applies to this household." us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning gets the correct total of $14,317.32 ($11,439 SS on the $184,500 base plus $2,878.32 Medicare), but it submitted $12,240.44. Nothing in its derivation supports that figure, so the submitted value contradicts its own correct computation." @@ -6445,117 +6522,120 @@ us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals us,scenario_078,payroll_tax,minimax-m3,llm_error,missing_output,False,"It entered 0 even though the household has $198,505 in wages, which leaves out all employee FICA. The correct answer is 6.2% Social Security on $184,500 ($11,439) plus 1.45% Medicare on all wages ($2,878.32)." us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"It projected the 2026 wage base as $174,900 by inflating the 2024 figure, instead of using the actual 2026 base of $184,500. That understates SS tax by $595.20 ($10,843.80 vs $11,439)." us,scenario_078,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It capped Social Security at the 2024 base of $168,600 ($10,453.20) instead of the 2026 base of $184,500. It then submitted $15,622.81, which does not match its own components ($10,453.20 + $2,878.32 = $13,331.52)." -us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used a ~$2,700 Maryland standard deduction and an $800 exemption, giving about $195,005 of taxable income. It should have itemized $54,670 in mortgage interest and real estate taxes, less the $10.27 phaseout. A single filer with Maryland AGI above $150,000 gets no exemption at all, so the correct taxable income is $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its structure was right, but it allowed a $600 personal exemption, which is $0 for a single filer with Maryland AGI above $150,000. It also skipped the itemized deduction reduction of 7.5% of federal AGI over $200,000 ($10.27). Together these understated taxable income by $610.27, or $32.04 of tax at the 5.25% rate." -us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied a $3,200 Maryland standard deduction to reach about $195,305 of taxable income. It should have itemized: Maryland allows the $28,210 mortgage interest and the full $26,460 in property taxes, less a $10.27 phaseout. Its $7,558 also does not follow from its own taxable income figure under Maryland's brackets." -us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed Maryland tax and entered $0. The correct path is Maryland AGI of $198,505, minus $54,659.73 of itemized deductions and a $0 exemption, which gives $143,845.27 taxed at Maryland rates for $6,936.88." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning correctly reached $143,835 of taxable income and $6,936.59 of tax, missing only the $10.27 phaseout. It then dropped that result and submitted an unsupported $9,755, which is consistent with taxing almost all of Maryland AGI without the itemized deductions." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assumed Maryland itemizing follows a $10,000 SALT cap and used only ~$30,000 of itemized deductions. Maryland itemized deductions include the full $26,460 in real estate taxes plus $28,210 in mortgage interest; only state income taxes are removed. It then submitted $9,876 even though its own bracket calculation came to $8,277.78." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left the $1,632 refund in Maryland AGI ($200,137 instead of $198,505). It also capped property taxes at $10,000, which made itemized deductions ~$38,210 instead of $54,659.73. On top of that it used a wrong rate schedule with a 5.0% top bracket for $150k–$175k." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption from $200,137. It should have subtracted the $1,632 refund to get Maryland AGI and itemized $54,670 in mortgage interest and property taxes, less the $10.27 phaseout. The exemption is $0 for a single filer above $150,000." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached about $143,835 of taxable income using $54,670 of itemized deductions. It then guessed $9,200 instead of applying the brackets: $90 on the first $3,000, 4.75% to $100k, 5% to $125k and 5.25% on the rest, which gives about $6,937." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used federal AGI of $200,137 as Maryland AGI without subtracting the $1,632 state tax refund, which Maryland excludes. It also skipped the $10.27 itemized phaseout. That left taxable income at $145,467 instead of $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped real estate taxes at $10,000, giving $38,210 of itemized deductions instead of the full $54,670 less $10.27. It claimed a $3,200 exemption, which is $0 above $150,000 for a single filer. It also kept the $1,632 refund in Maryland AGI." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $2,400 standard deduction and a $3,200 exemption on $200,137. It should have itemized $54,659.73, subtracted the $1,632 refund from Maryland AGI, and allowed no exemption, since Maryland AGI is above $150,000." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Its Maryland AGI ($198,505) and $0 exemption were correct. But it cut the $54,670 in itemized deductions by about $10,027 to $44,642.60. Maryland's phaseout is only 7.5% of federal AGI over $200,000, which here is $10.27, so taxable income should be $143,845.27, not $153,862.40." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left the $1,632 refund in Maryland AGI and used a 3% phaseout instead of 7.5% of federal AGI over $200,000. It also allowed a $1,600 exemption, which is $0 above $150,000. These errors partly cancelled, giving taxable income of $143,871.11 instead of $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $9,070 corresponds to about $183,000 of Maryland taxable income, which means only a standard-deduction-sized amount was subtracted. It ignored the $54,670 itemized deduction for mortgage interest and real estate taxes, which brings taxable income down to $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added county/local income tax to the state amount, but this output covers state tax only and excludes local tax. The state tax alone, on $143,845.27 after itemizing, is $6,936.88." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached roughly the right taxable income ($143,835, missing only the $10.27 phaseout) but got the bracket arithmetic wrong. $90 + $4,607.50 + $1,250 + 5.25% × $18,835 = $6,936.34, not $6,886." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It explicitly used standard deductions. $9,876 corresponds to about $197,500 of taxable income, so it never itemized the $54,670 in mortgage interest and property taxes that Maryland allows." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It computed the state tax correctly at $6,936.34, then added a 3.2% county income tax ($4,602.72). That local tax belongs in a separate output and is excluded from state income tax before refundable credits." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly itemized with a $0 exemption, which gives about $143,845 of taxable income. Its $6,889 is about $48 short, which means it misapplied the brackets. The correct bracket tax is $90 + $4,607.50 + $1,250 + 5.25% × $18,845.27 = $6,936.88." -us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It kept the $1,632 refund in Maryland AGI, giving $145,467 of taxable income. It then mis-applied the brackets: the correct tax on $145,467 is $7,022, not $7,300.69. The right base is $143,845.27, which gives $6,936.88." -us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It capped property taxes at $10,000 (federal itemized of $38,210) and ran a circular state-tax addback. Maryland allows the full $26,460 since the 2026 cap is $40,400 and no state income tax was paid. It also used a flat 4.75% rate, ignoring the 5% and 5.25% brackets above $100,000 and $125,000." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It applied a standard allowance. $9,720 corresponds to about $195,000 of taxable income, so it never subtracted the $54,659.73 in Maryland itemized deductions for mortgage interest and real estate taxes." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"$10,120 is more than Maryland's own brackets produce on the full $198,505 of AGI. So it neither applied the $54,659.73 itemized deduction nor used the correct rate schedule, which gives $6,936.88 on $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It claimed a personal exemption, which is $0 for a single filer above $150,000. $6,511.18 corresponds to about $135,700 of taxable income, roughly $8,100 below the correct $143,845.27, so it took deductions beyond the $54,659.73 Maryland allows." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"$7,025 corresponds to about $145,500 of taxable income. That means it itemized from federal AGI of $200,137 without subtracting the $1,632 refund that Maryland excludes, and without the $10.27 phaseout." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used $145,467 of taxable income, starting from federal AGI of $200,137. It should have subtracted the $1,632 state tax refund to get Maryland AGI of $198,505 and applied the 7.5% phaseout ($10.27), for taxable income of $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. $9,200 corresponds to about $185,000 of taxable income, so only a small standard-type deduction was subtracted. It skipped the $54,659.73 itemized deduction, which brings taxable income to $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It took the $2,400 standard deduction from $200,137. It should have subtracted the $1,632 refund to get Maryland AGI and itemized $54,670 in mortgage interest and real estate taxes, less the $10.27 phaseout, for taxable income of $143,845.27 instead of $197,737." -us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It cut itemized deductions by 3% of AGI over $100,000 ($3,004), but Maryland's limit is 7.5% of federal AGI over $200,000 ($10.27). It also kept a $576 exemption that is fully phased out above $150,000, and left the $1,632 refund in Maryland AGI." -us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"There was no parseable value for state_income_tax_before_refundable_credits, so no Maryland tax figure was submitted." -us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It entered $0 with no derivation, so it never applied Maryland tax to $143,845.27 of taxable income (Maryland AGI of $198,505 minus $54,659.73 of itemized deductions). That income is taxed at $6,936.88." -us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a flat 4.75% on everything above $3,000, leaving out the 5% bracket ($100k–$125k) and the 5.25% bracket ($125k–$150k). It also kept the $1,632 refund in Maryland AGI, giving $145,467 of taxable income instead of $143,845.27." -us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a ~$2,550 standard deduction instead of itemizing the $54,670 in mortgage interest and real estate taxes. Its $10,933 is also higher than its own $9,927.78 bracket calculation on the larger base, which is internally inconsistent." -us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income less a Maryland standard deduction instead of Maryland AGI ($198,505) less Maryland itemized deductions ($54,659.73). It used a wrong bracket schedule (5% on $95,000 and 5.25% on everything above $100,000). It also subtracted an invented nonrefundable credit, when none applies." +us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used a capped MD standard deduction of about $2,700 and an $800 exemption, giving taxable income near $195,005. It should have itemized $54,670 of mortgage interest and real estate taxes, less the $10.27 phase-out, which leaves $143,845.27 of taxable income. The exemption is $0 above $150,000 AGI for a single filer." +us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It kept a $600 personal exemption, but a single filer with AGI above $150,000 gets no Maryland exemption. It also skipped the 7.5% reduction of itemized deductions on federal AGI over $200,000 ($10.27). Together these understate taxable income by $610.27, which leaves the tax $32.04 short." +us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied a $3,200 MD standard deduction for taxable income of about $195,305 instead of itemizing the uncapped $54,670 of mortgage interest and property taxes. Its $7,558 also doesn't match the brackets applied to its own $195,305 base, which would give about $9,800." +us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It said it did not compute Maryland income tax and entered $0. Itemizing gives $143,845.27 of taxable income, which produces $6,936.88 under the 2%–5.25% brackets." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Its own reasoning reached $6,936.59 on taxable income of $143,835, which matches the reference except for the $10.27 itemized phase-out. It then dropped that figure without justification and submitted $9,755. That number fits a standard-deduction calculation that throws away the $54,670 of itemized deductions." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It limited itemized deductions to about $30,000 by applying a $10,000 SALT cap. In 2026 the OBBBA cap is far above the $26,460 of property taxes, so the full amount is deductible. It also submitted $9,876, which doesn't match its own bracket result of $8,277.78." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It capped real estate taxes at $10,000, for $38,210 of itemized deductions instead of $54,670 less the $10.27 phase-out. It also started from federal AGI of $200,137 without the Maryland subtraction for the $1,632 state tax refund, and it used a wrong 5.0% marginal rate for $150k–$175k." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $2,400 standard deduction and a $3,200 exemption on $200,137. It should have itemized $54,659.73 against MD AGI of $198,505, after subtracting the state refund. The exemption is $0 for a single filer above $150,000 AGI." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached about $143,835 of taxable income after itemizing $54,670. It then reported $9,200 instead of running the brackets: $90 + $4,607.50 at 4.75% up to $100k + $1,250 at 5% + 5.25% on the excess over $125k, which comes to about $6,936." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It started from $200,137 and didn't apply the Maryland subtraction for the $1,632 state tax refund already in federal AGI, so taxable income came out to $145,467 instead of $143,845.27. It also skipped the $10.27 itemized phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped property taxes at $10,000 for $38,210 of itemized deductions, but in 2026 the full $26,460 is deductible. It also kept a $3,200 exemption that is fully phased out above $150,000 AGI, and it didn't subtract the $1,632 state refund from AGI." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $2,400 standard deduction and a $3,200 exemption on $200,137, for $194,537 of taxable income. It should have itemized $54,659.73 against MD AGI of $198,505 with no exemption." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It correctly started at $198,505 with a $0 exemption, but cut itemized deductions by about $10,027, down to $44,642.60. Maryland's phase-out is only 7.5% of federal AGI over $200,000, which is $10.27, so itemized deductions are $54,659.73." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used 3% instead of 7.5% for the itemized phase-out and kept a $1,600 exemption, but the exemption is $0 above $150,000. It also didn't subtract the $1,632 state refund from AGI. These errors partly cancel, which is why its $6,938.23 lands close but still misses." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no specific steps. Its $9,070 matches taxing roughly $185k–$190k under the brackets, which means it didn't itemize the $54,670 of mortgage interest and property taxes that brings taxable income down to $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added county/local income tax to the state figure, even though the requested output excludes local income tax. The state-only tax on $143,845.27 is $6,936.88." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached $143,835 of taxable income but got about $50 too little from the brackets. Tax on that base is $90 + $4,607.50 + $1,250 + 5.25% × $18,835 = $6,936.34. It also skipped the $10.27 itemized phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It applied a standard deduction instead of itemizing the $54,670 of mortgage interest and real estate taxes, which taxed income near $197k at 5.5% instead of $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It correctly computed $6,936.34 of state tax, then added a 3.2% county tax ($4,602.72). The output excludes local income tax." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It itemized correctly with a $0 exemption but got about $48 too little from the brackets. Tax on $143,845.27 is $90 + $4,607.50 + $1,250 + 5.25% × $18,845.27 = $6,936.88." +us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It didn't subtract the $1,632 state refund, so its taxable income was $145,467. It then got $7,300.69 from the brackets even though that base gives $7,022.02. It also wrongly claimed the SALT cap expires in 2026." +us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used $38,210 of itemized deductions by applying a $10,000 SALT cap to property taxes, but all $26,460 is deductible for Maryland in 2026. It also made up a circular tax-on-tax calculation at a flat 4.75%." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It applied only a standard allowance to wages, which taxes roughly $195k. It should have itemized $54,659.73 of mortgage interest and property taxes against MD AGI of $198,505." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $10,120 is above the tax on the full $198,505 with no deductions at all. It didn't apply the $54,659.73 of itemized deductions that leave $143,845.27 of taxable income." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $6,511.18 matches about $135,700 of taxable income, roughly $8,100 below the correct base. It claimed a personal exemption, which is $0 above $150,000 AGI, and allowed more than the $54,659.73 of itemized deductions." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $7,025 matches taxing $145,467, which is federal AGI of $200,137 minus $54,670. It didn't subtract the $1,632 state refund from AGI and skipped the $10.27 itemized phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It taxed $145,467 by starting from federal AGI that still included the $1,632 state tax refund, which Maryland subtracts. It also skipped the 7.5% phase-out ($10.27) on AGI over $200,000." +us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. Its $9,200 matches a standard-deduction calculation on roughly $189k of income instead of itemizing the $54,670 of mortgage interest and property taxes down to $143,845.27 of taxable income." +us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a $2,400 standard deduction on $200,137, for $197,737 of taxable income. It should have itemized $54,659.73 against MD AGI of $198,505, net of the refund." +us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used a made-up itemized limitation of 3% of AGI over $100,000 ($3,004); Maryland's rule is 7.5% of federal AGI over $200,000 ($10.27). It also kept a $576 exemption that should be $0 and didn't subtract the $1,632 refund." +us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No value was returned for state_income_tax_before_refundable_credits. +us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It reported $0 with no calculation. Maryland taxes $143,845.27 after itemizing, which gives $6,936.88." +us,scenario_078,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It kept an $800 exemption, but the exemption is $0 above $150,000 AGI. It also taxed income above $100,000 mainly at 4.75%, ignoring the 5% bracket ($100k–$125k) and the 5.25% bracket ($125k–$150k)." +us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied a flat 4.75% to all income above $3,000, ignoring the 5% and 5.25% brackets above $100,000 and $125,000. It also didn't subtract the $1,632 state refund from AGI." +us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used only a standard deduction and ignored $54,670 of itemized mortgage interest and property taxes. Its $10,933 is also higher than the $9,927.78 it computed on the full $198,505 with no deduction." +us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income instead of MD AGI minus Maryland itemized deductions. It used a made-up 5% bracket from $5k–$100k with 5.25% on everything above, and subtracted a nonrefundable credit this filer doesn't qualify for." us,scenario_078,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_079,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model tested only Arizona's income-based adult (MAGI) Medicaid rule and declared the head failed it, never applying the SSI-recipient category: the head's blindness, disability, and $7,560 of SSDI produce $5,286 of SSI, and Arizona is a 1634 state where SSI approval automatically confers AHCCCS/Medicaid with no separate income or resource test. Its own stated route also cannot yield 'No' — countable income at 0.35 x FPL sits far below Arizona's 138% FPL expansion limit — so the model both skipped the operative categorical pathway and misapplied the fallback one it invoked." us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicitly listed pathway or coverage input and returned 0 for lack of one, but SSI receipt is computed from the facts given (age 58, blind, disabled, $7,560 SSDI, $1,045 in resources, no other income), and that SSI receipt is itself the categorical Medicaid pathway in Arizona, which auto-enrolls SSI recipients. It compounded this by reading Indian Health Service coverage at interview as the head's coverage source; IHS is not minimum essential coverage and never displaces or disqualifies Medicaid eligibility." -us,scenario_079,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model named the SSDI waiting period but treated 'constant for the full year' as proof that it was met. Constant receipt through 2026 adds up to only 12 months. The months-of-receipt input is unlisted and so defaults to 0, which means the 24-month requirement fails and age 58 is below the 65 threshold." -us,scenario_079,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model said SSDI receipt alone gives Medicare and skipped the 24-month entitlement requirement. No months of prior receipt are listed, so the count defaults to 0 and Head (58) qualifies by neither disability nor age." -us,scenario_079,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model stated the 24-month SSDI rule correctly but then assumed Head had met it, even though no listed fact supports that. The prompt's defaults set the unlisted SSDI duration to 0, so the disability route fails and Head at 58 is under the age-65 route." -us,scenario_079,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model noted that SSDI Medicare 'typically' comes after a waiting period but granted eligibility without confirming that period. SSDI months are unlisted and default to 0, so the 24-month requirement is not met and Head is ineligible." -us,scenario_079,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt as enough for disability-based Medicare and never applied the 24-month entitlement requirement. With 0 months of receipt on record and Head aged 58, Head qualifies by neither disability nor age." -us,scenario_079,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model cited the 'at least 24 months' SSDI condition but concluded Head met it from the annual SSDI amount and the disability flag, neither of which shows how long Head has received benefits. The unlisted duration defaults to 0, so the condition fails." -us,scenario_079,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model counted being disabled plus current SSDI receipt as disability-based Medicare eligibility, leaving out the required 24 months of SSDI entitlement. No months of receipt are listed (so 0), and age 58 is below 65." -us,scenario_079,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model assumed Medicare eligibility from being disabled and receiving SSDI without testing the 24-month SSDI duration condition. That duration input is unlisted and defaults to 0, so Head does not qualify." -us,scenario_079,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model explicitly assumed Head had completed the 24-month waiting period, which is an unlisted fact the prompt says to treat as 0. Once that assumption is removed, the disability route fails and Head (58) is under the age-65 threshold." -us,scenario_079,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare eligibility and skipped the 24-month entitlement requirement. SSDI months default to 0 and Head is 58, so neither route applies." -us,scenario_079,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model gave Medicare purely because Head receives SSDI and never applied the 24-month SSDI waiting period. With no months of receipt listed (defaulting to 0) and Head aged 58, Head is ineligible." -us,scenario_079,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated the disability flag plus SSDI income as qualifying on their own, leaving out the requirement of 24 months of SSDI entitlement. The unlisted months input is 0, so the disability route fails." -us,scenario_079,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model said PolicyEngine grants Medicare to any disabled SSDI recipient under 65, but PolicyEngine's disability route also requires at least 24 months of SSDI receipt. That months input defaults to 0 here, so Head is ineligible." -us,scenario_079,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model based eligibility on Head being 58 and disabled, as if the disability flag alone gave Medicare. The rule requires 24 months of SSDI entitlement, and Head's months of receipt are unlisted and default to 0." -us,scenario_079,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt 'for the year' as meeting PolicyEngine's disability route, but that route requires at least 24 months of SSDI receipt. A full year gives only 12 months, and prior months default to 0." -us,scenario_079,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model concluded that receiving SSDI makes Head Medicare-eligible on the basis of disability, without applying the 24-month SSDI entitlement requirement. Head's months of receipt default to 0, so the requirement is not met." -us,scenario_079,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model cited a 'disability-based Medicare eligibility rule' but left out that rule's condition of 24 months of SSDI receipt. That condition fails because Head's months of receipt are unlisted and default to 0, and Head (58) is under 65." -us,scenario_079,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model treated being disabled and receiving SSDI as enough for disability-based Medicare, skipping the required 24-month entitlement period. With SSDI duration defaulting to 0 and Head aged 58, Head is ineligible." -us,scenario_079,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied PolicyEngine's disability-based Medicare route without its condition of at least 24 months of SSDI receipt. The months input is unlisted and defaults to 0, so the route fails, and age 58 fails the age-65 test." -us,scenario_079,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model took for granted that Head had passed the standard 24-month waiting period, but no months of SSDI receipt are listed. Under the prompt's defaults the duration is 0, so the waiting period is not satisfied." -us,scenario_079,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine makes any disabled SSDI recipient under 65 Medicare-eligible, leaving out the 24-month SSDI receipt requirement. With 0 months of receipt, Head does not qualify." -us,scenario_079,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model stated the 24-month entitlement rule and then applied it as if it were already met. Nothing in the facts establishes 24 months of receipt, and the unlisted duration defaults to 0, so Head is ineligible." -us,scenario_079,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model explicitly assumed 'at least 24 months' of SSDI receipt, which the prompt forbids because unlisted numeric inputs are 0. Without that assumption, Head (58) meets neither the disability route nor the age-65 route." -us,scenario_079,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model read the 'constant throughout the year' instruction as satisfying the 24-month SSDI requirement, but constant receipt within 2026 gives only 12 months. Prior months are unlisted and default to 0, so Head does not qualify." +us,scenario_079,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model said SSDI recipients qualify 'after the waiting period' and then treated that period as satisfied, citing the constant-status instruction. That instruction covers only the tax-benefit year. Months of SSDI receipt are unlisted and count as 0, so the 24-month requirement is not met and Head, at 58, is not eligible." +us,scenario_079,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated receiving Social Security disability benefits as automatically giving Medicare eligibility under 65. It never applied the 24-month SSDI entitlement requirement. With no months of receipt listed (so 0), Head is ineligible at 58." +us,scenario_079,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model named the 24-month SSDI requirement but assumed Head had already met it. The household facts give no SSDI duration, which counts as 0 months, so the requirement fails and Head, at 58, is not eligible." +us,scenario_079,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model said SSDI recipients qualify 'typically after the waiting period' but did not check that period. It concluded Head was eligible even though the unlisted months of SSDI receipt are 0, which is short of the required 24 months." +us,scenario_079,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as enough for disability-based Medicare. It skipped the 24-month entitlement requirement, which Head does not meet because the unlisted duration of receipt is 0. At 58, Head also fails the age-65 test." +us,scenario_079,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model cited the at-least-24-months-of-SSDI rule but assumed without support that Head met it. The household facts give no months of receipt, which counts as 0, so Head, at 58, is not Medicare eligible." +us,scenario_079,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,The model gave disability-based Medicare eligibility because Head is disabled and receives SSDI. It never applied the 24-month SSDI waiting period. That period is unmet because months of receipt are unlisted and therefore 0. +us,scenario_079,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model assumed a disabled SSDI recipient is automatically Medicare eligible. It ignored the 24-month entitlement period, which Head fails with 0 months of receipt. Head is also under 65." +us,scenario_079,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model explicitly assumed Head had met the 24-month waiting period. That contradicts the instruction not to infer unlisted facts: the duration of SSDI receipt counts as 0, so Head, at 58, is not eligible." +us,scenario_079,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt alone as qualifying for Medicare. It omitted the 24-month SSDI entitlement requirement, which Head fails because months of receipt count as 0. At 58, Head is also below the age-65 threshold." +us,scenario_079,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model equated receiving SSDI with being Medicare eligible. It never checked the 24-month waiting period, which is unmet with 0 months of receipt. Head is 58, so the age route also fails." +us,scenario_079,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model concluded that disability plus SSDI receipt makes Head eligible. It skipped the requirement of 24 months of SSDI entitlement before disability-based Medicare begins, and the unlisted duration of receipt counts as 0." +us,scenario_079,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model claimed SSDI receipt qualifies a disabled person under 65 for Medicare. It did not apply the 24-month SSDI duration test, which Head fails because the unlisted months of receipt count as 0." +us,scenario_079,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model treated being age 58 and disabled as enough for Medicare. Disability-based Medicare requires 24 months of SSDI entitlement, and Head has 0 months listed, so Head is not eligible." +us,scenario_079,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model reasoned that receiving SSDI 'for the year' gives Medicare eligibility. Twelve months at most still falls short of the required 24 months of entitlement, and the unlisted duration of receipt counts as 0, so Head is ineligible." +us,scenario_079,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt as automatically conferring disability-based Medicare. It ignored the 24-month entitlement waiting period, which Head does not satisfy with 0 months of receipt at age 58." +us,scenario_079,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model invoked a disability-based Medicare rule but never checked its 24-month SSDI duration requirement. Head has 0 months of receipt and is under 65, so Head is not eligible." +us,scenario_079,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model took disability plus SSDI receipt as supporting Medicare eligibility. It omitted the 24-month SSDI entitlement requirement, which fails because the unlisted months of receipt count as 0." +us,scenario_079,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model asserted eligibility under disability-based Medicare rules without applying the 24-month SSDI waiting period. Head, at 58 with 0 months of SSDI receipt, meets neither the age route nor the disability route." +us,scenario_079,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model cited the 24-month waiting period but treated it as already satisfied. No SSDI duration is listed, so it counts as 0 months, and Head is not Medicare eligible at 58." +us,scenario_079,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model claimed SSDI receipt alone makes a disabled person under 65 Medicare eligible. It ignored the 24-month entitlement requirement, which Head fails with 0 months of receipt." +us,scenario_079,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model named the 24-month entitlement period but assumed Head had completed it. The unlisted duration of SSDI receipt counts as 0, so the period is unmet and Head, at 58, is ineligible." +us,scenario_079,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model explicitly assumed at least 24 months of SSDI receipt. That violates the instruction to treat unlisted numeric inputs as 0: with 0 months of receipt and age 58, Head is not Medicare eligible." +us,scenario_079,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI receipt as meeting the disability-based Medicare test. It skipped the required 24 months of SSDI entitlement, and Head's unlisted months of receipt count as 0." +us,scenario_079,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model cited the rule that SSDI recipients become eligible after 24 months of benefits, then used the constant-status instruction to assume those months had passed. That instruction covers only the current year; months of receipt count as 0, so Head, at 58, is ineligible." us,scenario_079,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_079,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630 of monthly SSDI and never computed the couple's $881 of monthly SSI ($1,491 couple FBR minus $610 of countable SSDI), so gross income was $1,511 short by $881. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, then gave a self-contradictory $4,560 that its own arithmetic does not support." -us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"Got the $1,511 income, the $209 standard deduction, and the $546 maximum right, but used the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households with costs above $35. That left net income at $1,276 instead of $1,157 and the benefit at about $163 a month instead of $198." -us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"Denied SNAP on assets by counting home equity and vehicle equity, but the home is an excluded resource and PolicyEngine's asset test counts only the $1,045.29 bank balance, which is well under the limit. It also left out the couple's SSI income." -us,scenario_079,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"Said $630 a month of SSDI was above the income limit. Disabled households skip the gross income test, and even with SSI included, net income of $1,157 is well below the $1,762.50 net limit (100% FPL), so the household qualifies for $198 a month." -us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Counted only $630 of SSDI and left out the couple's $881 a month of SSI. It also used a $219 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction, then arbitrarily lowered its own $420.50 result to $351.50." -us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Treated SSDI ($630 a month) as the only income and never added the $881 a month of SSI the disabled couple receives. On top of that, it guessed a net income of 'near $300' instead of applying $209 plus Arizona's $145 standard medical deduction to $1,511." -us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Left out the couple's $881 a month of SSI and used only $630 of SSDI as income. It also misapplied the medical deduction, treating $632 of annual expenses as a monthly figure, instead of applying Arizona's $145 standard medical deduction." -us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"Got the $1,511 income and $209 standard deduction right but deducted only the $26 actual medical excess, missing Arizona's $145 standard medical deduction for elderly/disabled households. Net income came out $119 too high, and the benefit $163 a month instead of $198." -us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Said the household receives SSI/SSDI but counted only the $630 of SSDI and never added the $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, along with outdated $204 and $535 figures." -us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Made up large shelter costs from the mortgage balance even though no mortgage payment or interest was given, and left out the $881 a month of SSI. It then reported $374 a month as the 'maximum allotment', which contradicts the actual $546 two-person maximum." -us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Counted only $630 of SSDI and left out the couple's $881 a month of SSI, so it used net income of $395 instead of $1,157. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction." -us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $228 standard deduction, a $549 maximum, and $876 of SSI instead of $209, $546, and $881." -us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"Counted vehicle equity above a $4,650 exclusion as a resource. Under Arizona's broad-based categorical eligibility (and PolicyEngine's asset test), vehicles are not counted, so countable assets are just the $1,045.29 bank balance and the household passes." -us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also used a $202 standard deduction, a $548 maximum, and SSI of $872 a month instead of $209, $546, and $881." -us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Used only $630 of SSDI as income and left out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, which gave it a $427 monthly benefit instead of $198." -us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also understated SSI at $805 instead of $881 and used outdated $198 standard-deduction and $535 maximum figures. -us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Treated having no rent or utility costs as making the benefit zero. Shelter costs only feed an optional deduction and are not required for eligibility; with $1,157 of net income the household receives $546 − $348 = $198 a month." -us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Claimed 30% of net income exceeds the maximum allotment. Net income after the $209 standard deduction and Arizona's $145 medical deduction is $1,157, so the household's contribution is only $348 against a $546 maximum, giving a regular $198 benefit rather than the minimum benefit." -us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,Deducted only $312 a year ($26 a month) of medical expense instead of Arizona's $145-a-month standard medical deduction. It also used a $200 standard deduction and a $560 maximum allotment instead of $209 and $546. -us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,asset_resource,False,"Declared the household ineligible on asset and income grounds without any computation. Countable assets are only $1,045.29 and net income of $1,157 is well below the net limit, so the household receives $198 a month." -us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $178 monthly benefit matches deducting only the actual $26 medical excess instead of Arizona's $145 standard medical deduction. With the $145 deduction, net income is $1,157 and the benefit is $198 a month." -us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $470 monthly benefit matches counting only the $630 of SSDI and leaving out the couple's $881 a month of SSI. Including SSI, net income is $1,157 and the benefit is $198." -us,scenario_079,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"Counted only the $432 of other medical expenses, dropped the $300 of over-the-counter costs, and deducted $12 a year instead of Arizona's $145-a-month standard medical deduction. It also used SSI of $840, a $204 standard deduction, and a $536 maximum instead of current values." -us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"Did not return a usable SNAP value. The correct derivation is $546 − 30% of $1,157 net income = $198 a month, or $2,376 a year." -us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used only the $7,560 of SSDI as gross income and left out the couple's $10,572 a year of SSI. It also deducted the $312 actual medical excess instead of Arizona's $1,740-a-year ($145-a-month) standard medical deduction." -us,scenario_079,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed SNAP is zero without dependents or shelter costs. Neither is required: the disabled couple has $1,157 of net income, below the $1,762.50 limit, and receives $198 a month." -us,scenario_079,snap,gpt-5.4-nano,llm_error,other,False,"Did no SNAP calculation and defaulted to zero, even though the facts are enough: $1,511 of SSDI+SSI income, the $209 standard deduction, and Arizona's $145 medical deduction give $198 a month." -us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction, and used a $204 standard deduction instead of $209. That left net income at $1,281 instead of $1,157 and the benefit at $161 instead of $198." -us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Awarded the full $546 maximum without subtracting 30% of net income. The household's $1,157 of net income produces a $348 contribution, so the benefit is $198 a month." -us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $163 monthly benefit matches deducting the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households, which overstated net income by $119." -us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $165 monthly benefit matches deducting only the actual $26 medical excess instead of Arizona's $145 standard medical deduction. With $145, net income is $1,157 and the benefit is $198." -us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"Deducted only $26 of excess medical expense instead of Arizona's $145 standard medical deduction, giving net income of $1,276 instead of $1,157. It also skipped rounding the 30% contribution up to a whole dollar." -us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,Its $395 net income comes from counting only the $630 of SSDI and leaving out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. -us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,"Got the $1,511 income and $209 standard deduction right but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction, which gave $163 a month instead of $198." -us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"Deducted the $26 actual medical excess instead of Arizona's $145 standard medical deduction, giving net income of $1,276 instead of $1,157 and an annual benefit of $1,958.40 instead of $2,376." -us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"Claimed SSDI income exceeds the SNAP limits without applying any test. Disabled households face only the net income test, and $1,157 of net income is far below the $1,762.50 limit, so the benefit is $198 a month." -us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used only the $7,560 of SSDI as income and left out the couple's $881 a month of SSI. It also deducted $312 a year of actual medical excess instead of Arizona's $145-a-month standard medical deduction." -us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used $876 of SSI and a $549 maximum instead of $881 and $546. -us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"Got the $1,511 income, $209 standard deduction, and $546 maximum right but deducted only $26 of excess medical expense instead of Arizona's $145 standard medical deduction, which gave $163.20 a month instead of $198." -us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"Deducted the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used the 2025 SSI rate ($840), a $204 standard deduction, and a $536 maximum instead of the 2026 values of $881, $209, and $546." -us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,Used only the $630 of monthly SSDI as gross income and left out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. -us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not return a usable SNAP value. The correct derivation is $546 − 30% of $1,157 net income = $198 a month, or $2,376 a year." -us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"Counted only $630 of SSDI and left out the $881 a month of SSI the disabled couple receives, so net income was $395 instead of $1,157. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction." -us,scenario_079,snap,minimax-m3,llm_error,other,False,"Admitted income is below the thresholds but returned zero to avoid 'inferring benefit receipt', which contradicts the prompt's instruction to assume program take-up. The eligible household receives $198 a month." -us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,Treated the $630 of SSDI as the household's only income and left out the couple's $881 a month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. -us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Left out the couple's SSI income and applied the $35 medical threshold annually instead of monthly. It also used a $598 maximum and a $198 standard deduction instead of $546 and $209, and missed Arizona's $145 standard medical deduction." -us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Applied a California-style SSI cash-out rule that bars SSI recipients from SNAP. Arizona has no such exclusion, and SSI receipt instead confers categorical eligibility, so the couple receives $198 a month." +us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model counted only the $630/month SSDI and left out the $881/month SSI the blind/disabled couple qualifies for, which sets gross income at $1,511. It also used the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction. Its final $4,560 doesn't match even its own $417.50/month calculation." +us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"The model correctly counted $1,511 of SSDI plus SSI, the $209 standard deduction, and the $546 maximum allotment. For medical costs, though, it deducted only the actual $26 excess ($61 − $35) instead of Arizona's $145 Standard Medical Deduction. That left net income at $1,276 instead of $1,157 and the benefit at about $163/month instead of $198." +us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model ruled the household out by counting home equity and vehicle equity as resources. In fact the home is fully excluded, and PolicyEngine's asset test counts only the $1,045 bank balance, which passes. The model also never computed the $881/month SSI that brings gross income to $1,511." +us,scenario_079,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said $630/month of SSDI is above the income limit. Disabled households skip the gross income test, and even with SSI included, net income of $1,157 is well under the 100% FPL net limit. The household is eligible and receives $198/month." +us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model counted only the $630 SSDI and left out the couple's $881/month SSI. It used an inflated $219 standard deduction and the actual $26 medical excess instead of Arizona's $145 SMD, then applied an unexplained 'adjustment' to reach $351.50/month." +us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model left out the $881/month SSI and treated $630 of SSDI as the household's only income. It then assumed an arbitrary net income of about $300 instead of applying the $209 standard deduction and Arizona's $145 Standard Medical Deduction to the correct $1,511 gross." +us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model left out the couple's $881/month SSI and counted only the $630 SSDI. It also misapplied the medical deduction, subtracting $35 from annual expenses, instead of using Arizona's flat $145 Standard Medical Deduction." +us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"The model correctly used $1,511 of gross income, the $209 standard deduction, and the $546 maximum. For medical costs, it deducted the actual $26 excess instead of Arizona's $145 Standard Medical Deduction, which overstated net income by $119 and gave $163/month instead of $198." +us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model said both members receive SSI/SSDI but put only $630 of SSDI into the calculation, dropping the $881/month SSI. It also used outdated $204 and $535 parameters and the actual $26 medical excess instead of Arizona's $145 SMD." +us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented large shelter costs from the mortgage balance, but no mortgage payment or interest was listed, so the shelter deduction is $0. It also left out the $881/month SSI and used a wrong $374 maximum allotment." +us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model counted only the $630/month SSDI and left out the $881/month SSI the couple qualifies for. It also used the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction, which made net income $395 instead of $1,157." +us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model included SSI, but at $876 instead of $881. It used a $228 standard deduction instead of $209, a $549 maximum instead of $546, and the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction." +us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"The model counted $5,050 of vehicle equity above a $4,650 exclusion and denied eligibility on resources. PolicyEngine's SNAP asset test counts only the $1,045 bank balance, which passes. The model also never computed the $881/month SSI." +us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model included SSI, slightly understated at $872/month. It used a $202 standard deduction and a $548 maximum, and deducted only the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction." +us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,The model counted only the $630 SSDI and left out the couple's $881/month SSI. It also used a $204 standard deduction and the actual $26 medical excess instead of $209 and Arizona's $145 SMD. +us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model understated SSI at $805 instead of $881/month. It used outdated $198 standard deduction and $535 maximum figures, and the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction." +us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the lack of rent or utility costs as making the benefit $0. A shelter deduction is not required for eligibility: $1,511 gross minus the $209 standard and $145 medical deductions leaves $1,157 net, which yields $198/month." +us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model claimed 30% of net income exceeds the maximum allotment, but 30% of $1,157 is $348, well below $546, so the benefit is $198/month. Its $276/year ($23/month) is also below the $24 monthly minimum." +us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"The model included SSI, but at $876 instead of $881/month. It used a $200 standard deduction instead of $209 and a $560 maximum instead of $546. It also deducted only $26/month of actual medical excess instead of Arizona's $145 Standard Medical Deduction." +us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model declared the household ineligible on assets and income. The household passes the asset test with $1,045 and the net income test with $1,157 net income, giving $198/month." +us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"The model included SSI ($18,096 total gross) but applied only the actual excess medical deduction instead of Arizona's $145 Standard Medical Deduction. That left the benefit at $178/month instead of $198." +us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's $470/month matches counting only the $630 SSDI and leaving out the couple's $881/month SSI. The correct calculation on $1,511 gross with the $209 standard and $145 AZ medical deductions yields $198/month." +us,scenario_079,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model used 2025-level SSI ($840 instead of $881/month) and a $536 maximum. It counted only the head's $432 other medical expense, dropping the OTC costs, and so missed that the $61/month total triggers Arizona's $145 Standard Medical Deduction." +us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so there was nothing to score." +us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model counted only the $7,560 SSDI and left out the couple's $10,572/year in SSI. It also used the actual $312/year medical excess instead of Arizona's $145/month Standard Medical Deduction." +us,scenario_079,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the absence of dependents and shelter costs as making SNAP $0. With $1,511 gross income and the $209 standard and $145 medical deductions, net income of $1,157 yields $546 − $348 = $198/month." +us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed there was not enough data to compute SNAP and returned $0. The listed SSDI, the SSI the couple qualifies for, and the listed medical expenses fully determine a $198/month benefit." +us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"The model correctly counted $1,511 of SSDI plus SSI but used a $204 standard deduction instead of $209. It also deducted only the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction, which gave $161/month instead of $198." +us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model awarded the full $546 maximum, which treats net income as zero. The household has $1,511/month of SSDI plus SSI, and after the $354 in deductions its $348 expected contribution must be subtracted." +us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"The model's $163/month matches $1,511 gross minus $209 and the actual $26 medical excess. It missed that Arizona substitutes a $145 Standard Medical Deduction, which lowers net income to $1,157 and raises the benefit to $198." +us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,The model included SSDI and SSI but used the actual excess medical deduction (about $26) instead of Arizona's $145 Standard Medical Deduction. That gave $165/month instead of $198. +us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"The model correctly used $1,511 gross, the $209 standard deduction, and the $546 maximum, but applied the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction. It also skipped rounding the contribution up to whole dollars." +us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's $395 net income comes from $630 SSDI alone, leaving out the couple's $881/month SSI. It also used the actual $26 medical excess instead of Arizona's $145 SMD." +us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,"The model correctly counted $1,511 of income and used $209 and $546. It deducted only the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction, which left the benefit at $163/month instead of $198." +us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"The model used $1,511 gross and the $209 standard deduction but only a $26 actual medical excess. Arizona's $145 Standard Medical Deduction lowers net income from $1,276 to $1,157 and gives $198/month." +us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"The model asserted that SSDI income exceeds SNAP limits. A disabled household is exempt from the gross test, and net income of $1,157 passes the net test, which produces a $198/month allotment." +us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model counted only the $7,560 SSDI and left out the couple's SSI ($881/month). It also used an inflated $2,520 standard deduction and the actual $312/year medical excess instead of Arizona's $145/month SMD." +us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,The model included SSI at $876 instead of $881 and used a $549 maximum instead of $546. It deducted only the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction. +us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"The model correctly computed $1,511 of income and used the $209 standard deduction and $546 maximum, but applied only $26 of actual excess medical costs. Arizona's $145 Standard Medical Deduction makes net income $1,157 and the benefit $198/month." +us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"The model used 2025 parameters: an $840 SSI figure from the $1,450 couple FBR, a $204 standard deduction, and a $536 maximum, instead of 2026 values ($881, $209, $546). It also deducted the actual $26 medical excess instead of Arizona's $145 SMD." +us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,The model counted only the $630/month SSDI and left out the $881/month SSI the blind/disabled couple qualifies for. It also used the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction. +us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so there was nothing to score." +us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model's net income of $395 comes from $630 SSDI alone, leaving out the couple's $881/month SSI. It also deducted the actual $26 medical excess instead of Arizona's $145 SMD." +us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model returned $0 because it refused to infer SNAP receipt, but the prompt explicitly assumes program take-up. On the listed facts, the eligible household receives $198/month." +us,scenario_079,snap,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model counted only the $630 monthly SSDI and left out the $881/month SSI. It also used the actual $26 medical excess instead of Arizona's $145 Standard Medical Deduction, which understated net income by $762." +us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,The model treated $630 of SSDI as the household's only income and left out the $881/month SSI. It also used the actual $26 medical excess instead of Arizona's $145 SMD. +us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model left out the $881/month SSI and used a wrong $598 maximum allotment. It also applied the $35 medical threshold annually instead of monthly, and never used Arizona's $145 Standard Medical Deduction." +us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied an SSI cash-out exclusion (California's former rule) that Arizona does not have. SSI recipients in Arizona are eligible for SNAP, and their SSI counts as unearned income, giving $198/month." us,scenario_079,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that the spouse's ""income and circumstances"" fail Arizona's adult Medicaid rules, inverting the test: the household's MAGI is 0.35 × FPL, far below Arizona's 138% FPL expansion-adult limit, so low income qualifies rather than disqualifies. It also skipped the pathway the engine used — the SSI-recipient category, which grants automatic Medicaid in Arizona (a 1634 state) to the blind/disabled spouse whose $0 countable income and $1,045 in resources (under the $2,000 individual limit) produce $5,286 of SSI — and treated an income screen as the sole route to adult coverage." us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to 0 on the ground that ""no additional Medicaid-eligibility inputs"" beyond disability and IHS coverage were supplied, treating Medicaid eligibility as requiring an explicit enrollment or income input rather than deriving it from the facts given. Blindness plus disability plus $0 earned and unearned income and $1,045 in countable resources produce SSI of $5,286, and SSI receipt in Arizona triggers the SSI-recipient category's automatic Medicaid; separately, 0.35 × FPL MAGI clears the 138% FPL expansion-adult test, and IHS coverage at interview is not minimum essential coverage that blocks eligibility." us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the spouse's disabled flag as enough for Medicare under 65, citing 'SSDI-linked disability eligibility'. However, the spouse gets no Social Security disability income; the $7,560 SSDI is the head's. No months of SSDI receipt are listed, so the 24-month SSDI entitlement requirement is not met and the spouse is not Medicare eligible." @@ -6596,46 +6676,47 @@ us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"It re us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"It misread the instruction not to infer benefit receipt as a reason to skip SSI, even though the prompt says to assume program take-up. Computing eligibility gives the blind and disabled couple $1,491 − $610 = $881/month." us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"It used $1,450/month ($17,400/year) as the 2026 couple rate, but $1,450 is the 2025 rate. The 2026 rate is $1,491/month ($17,892/year), so the benefit is $17,892 − $7,320 = $10,572, not $10,080." us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It combined two individual rates (about $947.65/month each) instead of using the single 2026 couple rate of $1,491/month. It also counted only $3,780 of the $7,560 SSDI and skipped the $20 exclusion, when the couple's countable income is $7,320/year. Together these errors inflated SSI to $18,963.60." -us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model said the Increased Excise Tax Credit 'requires filing with income' and needs earned income. In fact the credit is refundable and only requires federal AGI of $25,000 or less for MFJ; zero income qualifies. It should have given $25 × 2 exemptions = $50 instead of $0." -us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model wrongly said Arizona has no refundable individual income tax credit, and it treated having no taxable income as disqualifying. The refundable Increased Excise Tax Credit pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less, so this couple with $0 AGI gets $50." -us,scenario_079,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. It pays $25 per personal exemption to MFJ filers with federal AGI of $25,000 or less, and this couple has $0 AGI, so the credit is $50." -us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly worked out the Increased Excise Tax Credit as $25 per person, giving $50 for an MFJ couple with no dependents. It then threw that result away and submitted an unsupported 'best estimate' of $200. The correct total is $50." -us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model claimed the Increased Excise Tax Credit 'requires income tax filing thresholds not met here.' The credit has an income ceiling (federal AGI of $25,000 or less for MFJ), not a minimum income or filing threshold. With $0 AGI the couple qualifies for $25 × 2 = $50." -us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model assumed AZ refundable credits require earnings or qualifying dependents. The Increased Excise Tax Credit pays $25 for each personal exemption to low-AGI filers without either, so this couple gets $50." -us,scenario_079,state_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model overlooked Arizona's refundable Increased Excise Tax Credit, which is $25 per exemption for MFJ filers with federal AGI of $25,000 or less. This couple's AGI is $0, so the credit is $50." -us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model wrongly grouped the Increased Excise Tax Credit with nonrefundable credits and said it requires wages or AZ taxable income. The credit is refundable and only requires federal AGI of $25,000 or less for MFJ, so the couple gets $25 × 2 = $50." -us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model only checked the age-65/SSI-gated Property Tax Credit and missed the refundable Increased Excise Tax Credit. That credit has no age or SSI requirement and pays this $0-AGI MFJ couple $25 per exemption, or $50." -us,scenario_079,state_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model treated having no tax liability as ruling out refundable credits. Arizona's Increased Excise Tax Credit is refunded without any liability, at $25 per exemption for MFJ filers with AGI of $25,000 or less, giving $50 here." -us,scenario_079,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model assumed refundable credits need income or qualifying children. Arizona's Increased Excise Tax Credit is paid per personal exemption precisely to zero- and low-AGI filers, so this couple gets $50." -us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model checked dependent-child and property-tax credits but missed the refundable Increased Excise Tax Credit. That credit pays $25 for each spouse's exemption when MFJ federal AGI is $25,000 or less, for a total of $50." -us,scenario_079,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit, which pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less. This couple has $0 AGI, so the credit is $50." -us,scenario_079,state_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,The model treated zero state tax liability as ruling out refundable credits. The Increased Excise Tax Credit is refundable without any liability and pays $25 per exemption for a total of $50. -us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. Its only test is federal AGI of $25,000 or less for MFJ, which this couple meets at $0 AGI, so it pays $25 × 2 = $50." -us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. The couple qualifies through $0 federal AGI, well under the $25,000 MFJ cap, and receives $25 per exemption, or $50." -us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit, which pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less. That gives $50 for this couple with $0 AGI." -us,scenario_079,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model assumed refundable credits require tax liability or earnings. Arizona's Increased Excise Tax Credit requires neither and pays $25 per personal exemption to low-AGI filers, so this couple gets $50." -us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. The MFJ couple's $0 federal AGI is under the $25,000 cap, so the credit is $25 × 2 = $50." -us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit, which pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less. That gives $50 here." -us,scenario_079,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model claimed that under PolicyEngine rules, childless couples with no wages get no AZ refundable credits. In fact the Increased Excise Tax Credit requires no wages or children and pays $25 per personal exemption, giving $50 for this $0-AGI MFJ couple." -us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model treated zero taxable income as ruling out AZ refundable credits. The Increased Excise Tax Credit is refundable regardless of taxable income and pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less, for $50 here." -us,scenario_079,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"The model wrongly said Arizona's only refundable credit depends on federal EITC. Arizona has no EITC-linked credit; its refundable Increased Excise Tax Credit pays $25 per exemption to this $0-AGI MFJ couple, for $50." -us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model missed that the listed facts (Arizona residence, MFJ filing, two exemptions, $0 federal AGI) are enough for the refundable Increased Excise Tax Credit, which comes to $25 × 2 = $50." -us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required earned income or children for an AZ refundable credit. The Increased Excise Tax Credit is based only on personal exemptions and an AGI ceiling of $25,000 for MFJ, so this couple gets $50." -us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model assumed an AZ refundable credit needs liability, earned income, or children. The Increased Excise Tax Credit needs none of these and pays $25 per exemption to MFJ filers with federal AGI of $25,000 or less, for $50." -us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model missed that Arizona residency plus $0 federal AGI on an MFJ return qualifies for the refundable Increased Excise Tax Credit, which pays $25 for each spouse, or $50." -us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model required earnings or children for an AZ refundable credit. The Increased Excise Tax Credit pays $25 per personal exemption to low-AGI filers without either, giving $50 for this couple." -us,scenario_079,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"The model treated having no taxable income or dependents as ruling out AZ refundable credits. The Increased Excise Tax Credit is refundable at $25 per personal exemption when MFJ federal AGI is $25,000 or less, giving $50." -us,scenario_079,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"The model required earnings or dependent children. Arizona's refundable Increased Excise Tax Credit pays $25 per personal exemption to MFJ filers with federal AGI of $25,000 or less, so this couple gets $50." -us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model treated zero state tax liability as ruling out refundable credits. Arizona's refundable Increased Excise Tax Credit pays $25 per exemption regardless of liability, for $50 here." -us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model assumed AZ refundable credits need children and tax liability. The Increased Excise Tax Credit is paid per personal exemption to filers with AGI of $25,000 or less, giving $50 for this couple." -us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model invented an Arizona state EITC tied to the federal EITC; Arizona has none. It also missed the refundable Increased Excise Tax Credit, which pays this $0-AGI MFJ couple $25 × 2 = $50." -us,scenario_079,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model treated zero state tax liability as ruling out refundable credits. Arizona's Increased Excise Tax Credit is refunded in full at $25 per exemption to MFJ filers with AGI of $25,000 or less, giving $50." -us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model only checked the property-tax credit and earned-income-based credits, and missed the refundable Increased Excise Tax Credit. That credit needs no rent, property tax, or earnings and pays this $0-AGI MFJ couple $50." -us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model said no AZ refundable credit applies to a childless household with no liability. The Increased Excise Tax Credit pays $25 per personal exemption to exactly such low-AGI filers, so this couple gets $50." -us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model wrongly claimed Arizona has no state income tax; Arizona levies a 2.5% flat income tax. Its income tax system includes the refundable Increased Excise Tax Credit, which pays this couple $25 × 2 = $50." -us,scenario_079,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model correctly ruled out the age-65 Property Tax Credit but missed the refundable Increased Excise Tax Credit. That credit has no age test and pays $25 per exemption when MFJ federal AGI is $25,000 or less, giving $50." -us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated having no AZ taxable income or dependents as disqualifying. The refundable Increased Excise Tax Credit is $25 per personal exemption for MFJ filers with federal AGI of $25,000 or less, so this couple gets $50." -us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model missed Arizona's refundable Increased Excise Tax Credit. This MFJ couple's $0 federal AGI qualifies them for $25 per exemption, a total of $50." +us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"It wrongly claimed the Increased Excise Tax Credit requires filing with income. The credit is refundable and goes to joint filers with federal AGI of $25,000 or less, with no earned-income requirement. The couple's $7,560 SSDI income qualifies them for 2 × $25 = $50." +us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It wrongly stated that Arizona has no refundable individual income tax credit. The refundable Increased Excise Tax Credit pays $25 per filer to joint filers with AGI of $25,000 or less, even when Arizona taxable income is zero. The correct total is $50." +us,scenario_079,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It missed Arizona's refundable Increased Excise Tax Credit. That credit gives $25 each to the head and spouse because joint AGI is below the $25,000 limit, for a total of $50." +us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,It correctly worked out the Increased Excise Tax Credit as $25 × 2 = $50 for a joint couple with no dependents. It then dropped that result and submitted an unsupported $200. The correct value is the $50 it had already derived. +us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It wrongly treated the Increased Excise Tax Credit as needing income above a filing threshold. The credit is refundable and only requires joint federal AGI of $25,000 or less. The zero-taxable-income couple therefore receives $25 × 2 = $50." +us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"It assumed refundable Arizona credits require earnings or qualifying dependents. The Increased Excise Tax Credit pays $25 for each spouse to low-AGI joint filers, so the couple gets $50." +us,scenario_079,state_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,"It missed the refundable Arizona Increased Excise Tax Credit of $25 per filer, which applies because joint AGI is under $25,000. The correct total is $50." +us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It wrongly called the Increased Excise Tax Credit nonrefundable and tied it to taxable income. The credit is refundable and is paid to joint filers with AGI of $25,000 or less, so the couple gets $25 × 2 = $50." +us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It ruled out only the Property Tax Credit and missed the refundable Increased Excise Tax Credit. That credit has no age or SSI requirement and pays $25 for each spouse, for a total of $50." +us,scenario_079,state_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed that zero tax liability rules out refundable credits. Arizona's Increased Excise Tax Credit is paid regardless of liability to joint filers with AGI of $25,000 or less, giving $50." +us,scenario_079,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It assumed refundable credits need qualifying children. Arizona's Increased Excise Tax Credit counts the taxpayer and spouse at $25 each, so the low-AGI couple receives $50." +us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It looked only at dependent-based and property tax credits and missed the Increased Excise Tax Credit. That credit pays $25 for each spouse to joint filers with AGI of $25,000 or less, for a total of $50." +us,scenario_079,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It missed Arizona's refundable Increased Excise Tax Credit, which pays $25 for each spouse because joint AGI ($7,560 or less) is under $25,000. The correct total is $50." +us,scenario_079,state_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,It made the credit depend on having Arizona tax liability. The Increased Excise Tax Credit is fully refundable and pays $50 to this low-AGI joint couple even though their liability is zero. +us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It missed the Increased Excise Tax Credit, which Arizona pays to any joint filer with AGI of $25,000 or less at $25 per person. Here that is $50." +us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It said the household qualifies for no refundable credit. In fact the couple meets the $25,000 joint AGI limit for Arizona's Increased Excise Tax Credit, which pays $25 × 2 = $50." +us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It missed the refundable Arizona Increased Excise Tax Credit, which pays $25 for the head and $25 for the spouse to a low-AGI joint return. The correct total is $50." +us,scenario_079,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It required tax liability or earnings for a refundable credit. Arizona's Increased Excise Tax Credit requires neither and pays $50 to this couple because their AGI is under $25,000." +us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It missed Arizona's Increased Excise Tax Credit, a refundable $25 per taxpayer and spouse for joint filers with AGI of $25,000 or less, which comes to $50 here." +us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It missed the refundable Increased Excise Tax Credit, which the couple qualifies for because their joint AGI is far below $25,000. The credit gives $25 per spouse, for $50." +us,scenario_079,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It wrongly claimed that childless couples with no wages get no Arizona refundable credit. PolicyEngine's az_increased_excise_tax_credit pays $25 for each spouse under the $25,000 AGI limit, for $50." +us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"It made refundable credits depend on taxable income. Arizona's Increased Excise Tax Credit is paid at zero taxable income to joint filers with AGI of $25,000 or less, giving $25 × 2 = $50." +us,scenario_079,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It wrongly said Arizona has no refundable credit other than one tied to the EITC. The refundable Increased Excise Tax Credit does not depend on the EITC and pays this couple $25 each, for $50." +us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It missed the refundable Increased Excise Tax Credit. The credit is triggered just by joint federal AGI of $25,000 or less and pays $25 for each spouse, for $50." +us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed refundable credits need earned income or children. Arizona's Increased Excise Tax Credit counts the taxpayer and spouse at $25 each, so the low-AGI couple receives $50." +us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It required tax liability, earned income or children. Arizona's Increased Excise Tax Credit needs none of these and pays $25 per filer to joint returns with AGI of $25,000 or less, for $50." +us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It missed that the household's low AGI alone qualifies it for Arizona's refundable Increased Excise Tax Credit, which is $25 × 2 = $50." +us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It assumed refundable credits need earnings or children. The Increased Excise Tax Credit applies to any joint filer with AGI of $25,000 or less, giving this couple $50." +us,scenario_079,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"It treated zero taxable income and no dependents as ruling out a refundable credit. Arizona's Increased Excise Tax Credit pays $25 each to the taxpayer and spouse when AGI is under $25,000, for $50." +us,scenario_079,state_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It required earnings or dependent children. Arizona's refundable Increased Excise Tax Credit pays $25 for each spouse on a low-AGI joint return, for $50." +us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It made the credit depend on tax liability and missed the refundable Increased Excise Tax Credit, which pays this couple $50 because their joint AGI is under $25,000." +us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"It required qualifying children and tax liability. Arizona's Increased Excise Tax Credit counts the taxpayer and spouse at $25 each and is refundable, so the couple receives $50." +us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"It wrongly gave Arizona a state EITC and then stopped. It missed the refundable Increased Excise Tax Credit, which pays $25 per spouse to joint filers with AGI of $25,000 or less, for $50." +us,scenario_079,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,It treated zero state tax liability as ruling out refundable credits. Arizona's Increased Excise Tax Credit is paid in full to this low-AGI joint couple at $25 × 2 = $50. +us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"It looked only at the property tax refund and earned-income credits. It missed the Increased Excise Tax Credit, which needs no rent, property tax or earnings and pays $25 per spouse under the $25,000 AGI limit, for $50." +us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"It wrongly said Arizona has no refundable credit for a childless couple with zero liability. The Increased Excise Tax Credit pays $25 for each spouse, for $50." +us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It falsely stated that Arizona has no state income tax. Arizona does levy an individual income tax, and its refundable Increased Excise Tax Credit pays this low-AGI joint couple $25 × 2 = $50." +us,scenario_079,state_refundable_credits,muse-wonder,llm_error,categorical_eligibility,False,"It missed Arizona's refundable Increased Excise Tax Credit. The couple qualifies because their joint AGI is under $25,000, earning $25 per filer, or $50." +us,scenario_079,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It checked only the age-65 Property Tax Credit and said no other credit applies at $0 liability. The refundable Increased Excise Tax Credit has no age requirement and pays this couple $25 each, for $50." +us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It made refundable credits depend on taxable income and dependents. Arizona's Increased Excise Tax Credit counts the taxpayer and spouse at $25 each when AGI is under $25,000, for $50." +us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It missed the refundable Arizona Increased Excise Tax Credit, which pays $25 for each spouse on a joint return with AGI of $25,000 or less, for a total of $50." us,scenario_080,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -6660,53 +6741,54 @@ us,scenario_080,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F us,scenario_080,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_080,snap,claude-fable-5,reference_engine_defect,categorical_eligibility,False,"Correctly used the utility allowance from the energy subsidy to bring net income to $0, but applied a $293/month one-person maximum allotment for all 12 months. The correct maximum is $298/month for January–September and $304.68/month for October–December." -us,scenario_080,snap,claude-fable-5.1,reference_engine_defect,categorical_eligibility,False,"Used $298/month for all 12 months and missed that October–December 2026 fall in FY2027, when the one-person maximum rises to $304.68. That leaves out $20.04. It also dropped the $300/month financial assistance from countable income; net income still comes out to $0 only because of the heat-and-eat utility allowance." -us,scenario_080,snap,claude-haiku-4.5,reference_engine_defect,categorical_eligibility,False,"Applied a $2,750 SNAP resource limit and treated the $3,600 financial assistance as a resource. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test, so the household gets the full $3,596.04 maximum." -us,scenario_080,snap,claude-haiku-5.5,reference_engine_defect,categorical_eligibility,False,"Denied SNAP because the $19,828 in liquid assets exceeded the resource limit. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program waives the asset test entirely." -us,scenario_080,snap,claude-opus-4.7,reference_engine_defect,categorical_eligibility,False,"Reached $0 net income (by excluding the $300/month financial assistance instead of using the utility allowance) but used a $293/month maximum allotment ×12. The FY2026 one-person maximum is $298, rising to $304.68 from October." -us,scenario_080,snap,claude-opus-4.8,reference_engine_defect,categorical_eligibility,False,"Applied the roughly $4,500 resource limit for households with an elderly or disabled member and found the household ineligible. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program removes the asset test." -us,scenario_080,snap,claude-opus-5,reference_engine_defect,categorical_eligibility,False,"Its $218.50/month implies subtracting about 30% of a positive net income from the maximum, so it never applied the $857 heat-and-eat utility allowance from the $2,000 energy subsidy. That allowance plus the standard deduction brings net income to $0, which yields the full $298/$304.68 monthly maximum." -us,scenario_080,snap,claude-opus-5.5,reference_engine_defect,categorical_eligibility,False,"Explicitly applied the FY2026 $298 maximum to all 12 months and ignored the October 2026 cost-of-living update. That update raises the one-person maximum to $304.68 for October–December, so the annual total is $3,596.04, not $3,576." -us,scenario_080,snap,claude-sonnet-4.6,reference_engine_defect,categorical_eligibility,False,Correctly used the utility allowance to bring net income to $0 but used the FY2025 one-person maximum of $292/month ×12. The correct figures are $298/month for January–September and $304.68 for October–December. -us,scenario_080,snap,claude-sonnet-5,reference_engine_defect,categorical_eligibility,False,"Failed the household on a $4,250 disabled-household asset limit. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program waives the asset test, so the $19,828 bank balance does not matter." -us,scenario_080,snap,claude-sonnet-5.5,reference_engine_defect,categorical_eligibility,False,"Stated there were no shelter costs and never applied Pennsylvania's $857 utility allowance, which the $2,000 energy subsidy confers through heat-and-eat. It also counted the $924 in capital gains as income, leaving $169 of net income instead of $0, then subtracted 30% of it." -us,scenario_080,snap,deepseek-v4-flash-0731,reference_engine_defect,categorical_eligibility,False,"Counted the $924 in capital gains and applied only the standard deduction, leaving $170 of net income. It skipped the $857 heat-and-eat utility allowance triggered by the energy subsidy, which brings net income to $0 and yields the full maximum allotment." -us,scenario_080,snap,deepseek-v4-pro,reference_engine_defect,categorical_eligibility,False,"Correctly excluded the energy subsidy from income but missed that it triggers Pennsylvania's $857 utility allowance through heat-and-eat, so it left $174 of net income. It also used the outdated $292 maximum instead of $298/$304.68." -us,scenario_080,snap,deepseek-v4-pro-0813,reference_engine_defect,categorical_eligibility,False,"Assumed there was no shelter deduction and left $168 of net income. It missed that the $2,000 energy subsidy entitles the household to the $857 heat-and-eat utility allowance, which brings net income to $0. It also counted capital gains as SNAP income." -us,scenario_080,snap,deepseek-v4.1-flash,reference_engine_defect,categorical_eligibility,False,Denied SNAP on the asset limit for disabled-member households. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test. -us,scenario_080,snap,gemini-3-flash-preview,reference_engine_defect,categorical_eligibility,False,Correctly used the utility allowance to bring net income to $0 but used a $291/month maximum ×12. The one-person maximum is $298 for January–September and $304.68 for October–December. -us,scenario_080,snap,gemini-3.1-flash-lite-preview,reference_engine_defect,categorical_eligibility,False,Cited the household's assets and lack of income as grounds for $0. It missed that Pennsylvania's categorical eligibility waives the asset test and that the $0 net income it would have found yields the maximum benefit. -us,scenario_080,snap,gemini-3.1-pro-preview,reference_engine_defect,categorical_eligibility,False,Correctly reached $0 net income through the utility allowance but used the FY2025 $292/month maximum ×12. The correct maximums are $298 for January–September and $304.68 for October–December. -us,scenario_080,snap,gemini-3.5-flash,reference_engine_defect,categorical_eligibility,False,"Correctly reached $0 net income through the energy-assistance utility allowance but applied the FY2025 $292/month maximum for all 12 months. The FY2026 maximum is $298, and $304.68 applies from October." -us,scenario_080,snap,gemini-3.5-flash-lite,reference_engine_defect,categorical_eligibility,False,"Zeroed the benefit because of the household's assets. It missed that Pennsylvania's broad-based categorical eligibility removes the asset test and that the household's net income is $0, which yields the maximum allotment." -us,scenario_080,snap,gemini-3.6-flash,reference_engine_defect,categorical_eligibility,False,Correctly applied broad-based categorical eligibility and the heating utility allowance but used a $292/month maximum ×12. The correct figures are $298/month for January–September and $304.68 for October–December. -us,scenario_080,snap,gemini-3.7-flash,reference_engine_defect,categorical_eligibility,False,"Applied the $298 FY2026 maximum to all 12 months. It missed the FY2027 increase to $304.68/month for October–December 2026, which adds $20.04 to the annual total." -us,scenario_080,snap,gemini-3.8-flash,reference_engine_defect,categorical_eligibility,False,Correctly used broad-based categorical eligibility and the heating utility allowance to reach $0 net income but used the outdated $292/month maximum ×12. The correct maximums are $298 and $304.68. -us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score." -us,scenario_080,snap,glm-5.3,reference_engine_defect,categorical_eligibility,False,"Recognized the energy subsidy as LIHEAP-type assistance but used it only for categorical eligibility, not to trigger the $857 utility allowance. It therefore left about $170 of net income and also used an outdated $290 maximum." -us,scenario_080,snap,gpt-5.4-mini,reference_engine_defect,categorical_eligibility,False,"Claimed the facts were insufficient and returned $0. In fact the one-person household with $300/month assistance qualifies through Pennsylvania's broad-based categorical eligibility and the heat-and-eat utility allowance, which brings net income to $0 and pays the maximum allotment." -us,scenario_080,snap,gpt-5.4-nano,reference_engine_defect,categorical_eligibility,False,"Set SNAP to $0 for lack of earned income and expense details. SNAP needs no earned income: the household's $301 of monthly income, Pennsylvania's broad-based categorical eligibility, and the $857 utility allowance from the energy subsidy produce the full maximum benefit." -us,scenario_080,snap,gpt-5.5,reference_engine_defect,categorical_eligibility,False,"Stated there was no shelter deduction and left $96 of net income. It missed that the $2,000 energy subsidy triggers the $857 heat-and-eat utility allowance, which brings net income to $0. It also did not apply the $304.68 October–December maximum." -us,scenario_080,snap,gpt-5.6-luna,reference_engine_defect,categorical_eligibility,False,"Counted the $924 in capital gains as SNAP income and applied no shelter deduction. It missed the $857 heat-and-eat utility allowance from the energy subsidy, which brings net income to $0 and yields the full maximum allotment." -us,scenario_080,snap,gpt-5.6-sol,reference_engine_defect,categorical_eligibility,False,"Reduced income only by the standard deduction and subtracted 30% of the remainder, reaching $270/month. It never applied the heat-and-eat utility allowance from the $2,000 energy subsidy, which brings net income to $0." -us,scenario_080,snap,gpt-5.6-terra,reference_engine_defect,categorical_eligibility,False,"Applied the SNAP resource limit to the $19,828 bank balance. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test." -us,scenario_080,snap,gpt-6-astra,reference_engine_defect,categorical_eligibility,False,Correctly used categorical eligibility and the heating utility allowance to reach the maximum benefit but paid $298 for all 12 months. It missed the October 2026 increase to $304.68/month for the FY2027 months. -us,scenario_080,snap,gpt-6-luna,reference_engine_defect,categorical_eligibility,False,"Applied only the standard deduction to the $300/month assistance and subtracted 30% of the remainder. It missed that the energy subsidy triggers the $857 heat-and-eat utility allowance, which brings net income to $0 and pays the full maximum." -us,scenario_080,snap,gpt-6-sol,reference_engine_defect,categorical_eligibility,False,Counted $940 of investment income (including the $924 in capital gains) and applied only the $209 standard deduction. It omitted the $857 heat-and-eat utility allowance that brings net income to $0. -us,scenario_080,snap,gpt-6.1-sol,reference_engine_defect,categorical_eligibility,False,"Subtracted 30% of $2,032 in annual net income from $3,576. That income figure counts capital gains and skips the $857 heat-and-eat utility allowance, which brings net income to $0. It also missed the $304.68 maximum for October–December." -us,scenario_080,snap,grok-4.3,reference_engine_defect,categorical_eligibility,False,"Treated the absence of shelter expenses as producing a $0 benefit, which reverses how the formula works: lower net income raises the benefit. It also missed that the energy subsidy confers the $857 utility allowance, so net income is $0 and the household receives the maximum allotment." -us,scenario_080,snap,grok-4.5,reference_engine_defect,categorical_eligibility,False,Correctly applied broad-based categorical eligibility and the utility allowance to reach $0 net income but used the $292/month FY2025 maximum ×12. The correct maximums are $298 for January–September and $304.68 for October–December. -us,scenario_080,snap,grok-4.6,reference_engine_defect,categorical_eligibility,False,Correctly reached $0 net income through the heating utility allowance but used a $292/month maximum ×12. The correct one-person allotments are $298 for January–September and $304.68 for October–December. -us,scenario_080,snap,grok-4.7,reference_engine_defect,categorical_eligibility,False,"Used $298 for all 12 months, missing the FY2027 increase to $304.68/month for October–December 2026. It also excluded the $300/month assistance from income, though net income still comes out to $0 because of the utility allowance." -us,scenario_080,snap,grok-build-0.1,reference_engine_defect,categorical_eligibility,False,"Applied the $4,250 disabled-household asset limit and found the household ineligible. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program waives the asset test." -us,scenario_080,snap,inkling,reference_engine_defect,categorical_eligibility,False,Denied SNAP on the resource limit and wrongly tied eligibility to school-age children. It missed that Pennsylvania's broad-based categorical eligibility removes the asset test for this one-person household. -us,scenario_080,snap,kimi-k2.6,reference_engine_defect,categorical_eligibility,False,"Applied the federal resource limit for disabled-member households. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program removes the asset test, so the household gets the maximum allotment." -us,scenario_080,snap,kimi-k3,reference_engine_defect,categorical_eligibility,False,"Reached $0 net income (by excluding the financial assistance) and correctly waived the asset test, but used a $292/month maximum ×12. The correct allotments are $298 for January–September and $304.68 for October–December." -us,scenario_080,snap,minimax-m3,reference_engine_defect,categorical_eligibility,False,"Treated the $19,828 in assets (plus the assistance) as disqualifying. It missed that Pennsylvania's broad-based categorical eligibility waives the asset test and that net income is $0, which pays the maximum allotment." -us,scenario_080,snap,ox-alpha,reference_engine_defect,categorical_eligibility,False,"Applied the roughly $4,500 disabled-household asset limit. It missed that Pennsylvania's categorical eligibility through the TANF non-cash program removes the asset test entirely." -us,scenario_080,snap,qwen-3.7-max,reference_engine_defect,categorical_eligibility,False,"Applied the $2,750 general asset limit, ignoring both the higher limit for disabled households and, decisively, Pennsylvania's broad-based categorical eligibility, which waives the asset test." -us,scenario_080,snap,qwen3.8-max,reference_engine_defect,categorical_eligibility,False,"Denied SNAP because resources exceeded the limit. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program waives the asset test, and that a lack of earned income does not prevent SNAP eligibility." +us,scenario_080,snap,claude-fable-5,reference_engine_defect,categorical_eligibility,False,"It correctly applied broad-based categorical eligibility (BBCE) and the utility allowance triggered by the energy subsidy to get $0 net income. But it used a $293 one-person maximum allotment for all 12 months. The correct maximum is $298 for January–September and $304.68 for October–December, so $293×12 = $3,516 is too low." +us,scenario_080,snap,claude-fable-5.1,reference_engine_defect,categorical_eligibility,False,"It reached the maximum allotment, but by leaving out the $300/month financial assistance, which SNAP counts. The correct route to $0 net income is the $857 utility allowance triggered by the energy subsidy. It also multiplied $298 by 12 and missed the October 2026 increase to $304.68 for October–December." +us,scenario_080,snap,claude-haiku-4.5,reference_engine_defect,categorical_eligibility,False,"It wrongly counted the $3,600 financial assistance as a resource and applied a $2,750 asset limit. It never applied Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which waives the SNAP asset test entirely. That made the household look ineligible when it actually qualifies for the maximum allotment." +us,scenario_080,snap,claude-haiku-5.5,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because $19,828 in liquid assets exceeds the resource limit. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test. It also wrongly said there is no counted income, ignoring the $300/month financial assistance." +us,scenario_080,snap,claude-opus-4.7,reference_engine_defect,categorical_eligibility,False,"It wrongly excluded the $300/month financial assistance, but still landed on $0 net income and the maximum benefit. Its error was using a $293 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December." +us,scenario_080,snap,claude-opus-4.8,reference_engine_defect,categorical_eligibility,False,"It applied the roughly $4,500 elderly/disabled asset limit and denied SNAP. It missed Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which waives the asset test. With $0 net income after the standard deduction and utility allowance, the household gets the maximum allotment." +us,scenario_080,snap,claude-opus-5,reference_engine_defect,categorical_eligibility,False,"Its $2,622 works out to about $218.50/month, well below the maximum. It wrongly treated shelter deduction limits as reducing the benefit. In fact, the $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance, and the uncapped excess shelter deduction takes net income to $0, giving the full $298 and $304.68 monthly maximums." +us,scenario_080,snap,claude-opus-5.5,reference_engine_defect,categorical_eligibility,False,"It correctly reached $0 net income and the $298 maximum but applied that amount to all 12 months. It explicitly ignored the October 2026 cost-of-living update, which raises the October–December allotment to $304.68." +us,scenario_080,snap,claude-sonnet-4.6,reference_engine_defect,categorical_eligibility,False,"It correctly applied broad-based categorical eligibility, the utility allowance from the energy subsidy, and $0 net income. But it used the outdated FY2025 $292 maximum allotment for all 12 months instead of $298 for January–September and $304.68 for October–December." +us,scenario_080,snap,claude-sonnet-5,reference_engine_defect,categorical_eligibility,False,"It computed near-zero net income, then disqualified the household under the $4,250 elderly/disabled asset limit. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program waives the asset test entirely." +us,scenario_080,snap,claude-sonnet-5.5,reference_engine_defect,categorical_eligibility,False,"It wrongly said there are no shelter costs. The $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance, and the resulting excess shelter deduction takes net income to $0. Instead it subtracted 30% of a $169 net income from $298 and also counted capital gains, which SNAP excludes." +us,scenario_080,snap,deepseek-v4-flash-0731,reference_engine_defect,categorical_eligibility,False,"It applied only the standard deduction and missed that the $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance. That allowance creates an excess shelter deduction that cuts net income to $0. It also counted the $924 of capital gains as SNAP income, leaving a $170 net income and a reduced benefit." +us,scenario_080,snap,deepseek-v4-pro,reference_engine_defect,categorical_eligibility,False,"It wrongly concluded that the energy subsidy only counts as excluded income and that no shelter deduction applies. In fact, the subsidy triggers Pennsylvania's $857 utility allowance, which takes net income to $0. It also used an outdated $292 maximum instead of $298 for January–September and $304.68 for October–December." +us,scenario_080,snap,deepseek-v4-pro-0813,reference_engine_defect,categorical_eligibility,False,"It wrongly assumed no shelter deduction. The $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance, which takes net income to $0 rather than $168. It also counted capital gains as SNAP income, so it took 30% off a $300 maximum instead of paying the full allotment." +us,scenario_080,snap,deepseek-v4.1-flash,reference_engine_defect,categorical_eligibility,False,"It wrongly denied SNAP because $19,828 in assets exceeds the disabled-household resource limit. It missed Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which waives the asset test." +us,scenario_080,snap,gemini-3-flash-preview,reference_engine_defect,categorical_eligibility,False,It correctly used the utility allowance from the energy subsidy to reach $0 net income. But it applied a $291 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December. +us,scenario_080,snap,gemini-3.1-flash-lite-preview,reference_engine_defect,categorical_eligibility,False,"It wrongly treated assets and low income as reasons for zero SNAP. Pennsylvania's broad-based categorical eligibility waives the asset test. Low income raises the benefit: with $0 net income after the standard deduction and utility allowance, the household gets the maximum one-person allotment." +us,scenario_080,snap,gemini-3.1-pro-preview,reference_engine_defect,categorical_eligibility,False,It correctly found that the utility allowance from the energy subsidy takes net income to $0. But it used a $292 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December. +us,scenario_080,snap,gemini-3.5-flash,reference_engine_defect,categorical_eligibility,False,"It reached $0 net income through broad-based categorical eligibility and the utility allowance, though with outdated $198 standard deduction and $600 allowance figures. The final error was a $292 maximum allotment for all 12 months instead of $298 for January–September and $304.68 for October–December." +us,scenario_080,snap,gemini-3.5-flash-lite,reference_engine_defect,categorical_eligibility,False,"It wrongly treated the financial assistance and assets as disqualifying. Pennsylvania's broad-based categorical eligibility waives the asset test, and the $301 monthly countable income falls to $0 net after the standard deduction and $857 utility allowance, which yields the maximum allotment." +us,scenario_080,snap,gemini-3.6-flash,reference_engine_defect,categorical_eligibility,False,It correctly applied broad-based categorical eligibility and the heating utility allowance to reach $0 net income. But it used a $292 one-person maximum for all 12 months instead of $298 for January–September and $304.68 for October–December. +us,scenario_080,snap,gemini-3.7-flash,reference_engine_defect,categorical_eligibility,False,"It reached the $298 maximum but multiplied it by 12. It missed the October 2026 cost-of-living increase to $304.68 for October–December, which brings the annual total to $3,596.04." +us,scenario_080,snap,gemini-3.8-flash,reference_engine_defect,categorical_eligibility,False,It correctly used broad-based categorical eligibility and the heating utility allowance to reach $0 net income. But it applied an outdated $292 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December. +us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"It returned no SNAP value and no reasoning, so there is no answer to score." +us,scenario_080,snap,glm-5.3,reference_engine_defect,categorical_eligibility,False,It recognized the energy subsidy as LIHEAP-type assistance but never used it to trigger Pennsylvania's $857 utility allowance. That allowance's excess shelter deduction takes net income to $0. Instead it took 30% of about $170 net income off a $290 maximum. +us,scenario_080,snap,gpt-5.4-mini,reference_engine_defect,categorical_eligibility,False,"It wrongly treated the facts as too thin to compute SNAP. A one-person household with $301/month countable income clears the gross test under broad-based categorical eligibility. After the standard deduction and $857 utility allowance, its net income is $0, which yields the maximum allotment." +us,scenario_080,snap,gpt-5.4-nano,reference_engine_defect,categorical_eligibility,False,"It wrongly set SNAP to $0 because there was no earned income. SNAP pays the most to households with little or no income. Here the household has $0 net income after the standard deduction and the utility allowance triggered by the energy subsidy, so it gets the maximum one-person allotment." +us,scenario_080,snap,gpt-5.5,reference_engine_defect,categorical_eligibility,False,"It wrongly assumed no shelter deduction. The $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance, which takes net income from $96 to $0. Instead of the full $298 and $304.68 monthly maximums, it took 30% of $96 off $298." +us,scenario_080,snap,gpt-5.6-luna,reference_engine_defect,categorical_eligibility,False,"It applied only the standard deduction and missed the $857 utility allowance triggered by the energy subsidy, which takes net income to $0. It also counted the $924 of capital gains in its $4,540 countable income, so it subtracted 30% of net income from the maximum." +us,scenario_080,snap,gpt-5.6-sol,reference_engine_defect,categorical_eligibility,False,"It reduced income only by the standard deduction and missed that the $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance. The resulting excess shelter deduction takes net income to $0, so the benefit is the full maximum, not a $270 reduced allotment." +us,scenario_080,snap,gpt-5.6-terra,reference_engine_defect,categorical_eligibility,False,"It wrongly denied SNAP because $19,828 exceeds the disabled-household resource limit. It missed Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which waives the asset test." +us,scenario_080,snap,gpt-6-astra,reference_engine_defect,categorical_eligibility,False,"It correctly applied broad-based categorical eligibility and the heating utility allowance to reach $0 net income. But it paid $298 for all 12 months, missing the October 2026 increase to $304.68 for October–December." +us,scenario_080,snap,gpt-6-luna,reference_engine_defect,categorical_eligibility,False,"It applied only the standard deduction and subtracted 30% of the remaining net income. It missed that the $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance, which takes net income to $0 and yields the full maximum allotment." +us,scenario_080,snap,gpt-6-sol,reference_engine_defect,categorical_eligibility,False,"It counted the $940 of investment income (SNAP counts only the $16 of dividends, not the capital gains) and applied only the $209 standard deduction. It missed the $857 utility allowance triggered by the energy subsidy, which takes net income to $0." +us,scenario_080,snap,gpt-6.1-sol,reference_engine_defect,categorical_eligibility,False,"It got broad-based categorical eligibility right but left $2,032 of annual net income after only the standard deduction. It missed that the energy subsidy triggers Pennsylvania's $857 utility allowance, whose excess shelter deduction takes net income to $0. It also held the maximum at $298 instead of rising to $304.68 in October–December." +us,scenario_080,snap,grok-4.3,reference_engine_defect,categorical_eligibility,False,"It wrongly concluded that having no shelter or utility expenses makes SNAP $0. A household with little income gets close to the maximum even without shelter costs. Here, the $2,000 energy subsidy triggers Pennsylvania's $857 utility allowance, which takes net income to $0 and yields the full allotment." +us,scenario_080,snap,grok-4.5,reference_engine_defect,categorical_eligibility,False,It correctly applied broad-based categorical eligibility and the uncapped utility-allowance shelter deduction to reach $0 net income. But it used a $292 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December. +us,scenario_080,snap,grok-4.6,reference_engine_defect,categorical_eligibility,False,It correctly reached $0 net income through broad-based categorical eligibility and the heating utility allowance from the energy subsidy. But it applied a $292 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December. +us,scenario_080,snap,grok-4.7,reference_engine_defect,categorical_eligibility,False,"It wrongly excluded the $300/month financial assistance from countable income, though it still reached $0 net income. It then paid $298 for all 12 months, missing the October 2026 increase to $304.68 for October–December." +us,scenario_080,snap,grok-build-0.1,reference_engine_defect,categorical_eligibility,False,"It applied the $4,250 disabled-household asset limit and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program waives the asset test." +us,scenario_080,snap,inkling,reference_engine_defect,categorical_eligibility,False,"It counted bank assets and vehicle value against a disabled-household resource limit and also cited the absence of school-age children, which is irrelevant to SNAP. It missed Pennsylvania's broad-based categorical eligibility, which waives the asset test, so the household gets the maximum allotment." +us,scenario_080,snap,kimi-k2.6,reference_engine_defect,categorical_eligibility,False,"It applied the federal resource limit for households with a disabled member and denied SNAP. It missed Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which removes the asset test." +us,scenario_080,snap,kimi-k3,reference_engine_defect,categorical_eligibility,False,"It wrongly counted only the small investment income and ignored the $300/month financial assistance, though $0 net income still follows from the utility allowance. Its final error was a $292 monthly maximum for all 12 months instead of $298 for January–September and $304.68 for October–December." +us,scenario_080,snap,minimax-m3,reference_engine_defect,categorical_eligibility,False,"It wrongly treated $19,828 in assets plus the financial assistance as disqualifying. Pennsylvania's broad-based categorical eligibility waives the asset test, and the household's $0 net income after the standard deduction and $857 utility allowance yields the maximum allotment." +us,scenario_080,snap,muse-wonder,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because bank assets exceed the disabled-household resource limit. It missed Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which waives the asset test." +us,scenario_080,snap,ox-alpha,reference_engine_defect,categorical_eligibility,False,"It applied the roughly $4,500 disabled-household asset limit and denied SNAP despite very low income. It missed that Pennsylvania's broad-based categorical eligibility through the TANF non-cash program removes the asset test." +us,scenario_080,snap,qwen-3.7-max,reference_engine_defect,categorical_eligibility,False,"It applied a $2,750 asset limit and denied SNAP. It missed Pennsylvania's broad-based categorical eligibility through the TANF non-cash program, which waives the asset test entirely." +us,scenario_080,snap,qwen3.8-max,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because resources exceed the limit. It missed Pennsylvania's broad-based categorical eligibility, which waives the asset test, so the household's $0 net income yields the maximum one-person allotment." us,scenario_080,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, violating the required structured-output contract." us,scenario_080,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the prompt's general disability fact as establishing SSI disability, despite the applicable engine input is_ssi_aged_blind_disabled=False. Because the 41-year-old is neither SSI-aged, blind, nor SSI-disabled, the federal base-rate and unearned-income calculation does not apply and SSI is $0." us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived 100% Pennsylvania Tax Forgiveness and explicitly calculated that it reduces the $28.86 preliminary tax to $0, but then submitted the unreduced $28.86. It failed to carry its own nonrefundable-credit calculation into the output." @@ -6720,137 +6802,140 @@ us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_erro us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable output for state_refundable_credits, violating the required submission contract." us,scenario_080,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model misclassified Pennsylvania Schedule SP tax forgiveness as a refundable credit that pays out the $28.86 liability. Schedule SP is nonrefundable and only reduces state income tax, so it contributes $0 to state_refundable_credits." us,scenario_080,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It set the Massachusetts income tax in SALT to zero and deducted only the $5,643 property tax. That dropped the $8,484.41 estimated MA withholding that fits under the $40,400 OBBBA cap and overstated taxable income at about $151,550. It then raised its own $28,759 bracket result to $29,385 with no stated reason." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,230 for MA income tax in SALT, which is the final MA liability, instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. Itemized deductions came out about $254 too low, taxable income about $260 too high ($143,320 vs $143,059.47), and the tax about $63 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $14,600 standard deduction (the 2024 figure) and wrongly said the $16,900 mortgage interest did not justify itemizing. It skipped the $31,027.41 of itemized deductions (mortgage interest + property tax + MA income tax under the $40,400 SALT cap). It also deducted the $1,782 of unreimbursed employee expenses above the line, used 2024 brackets, and submitted a number that does not match its own $30,956.92." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $33,000 is a round guess with no derivation. It is higher than even the $30,514.85 tax under the $16,100 standard deduction, so it never applied the $31,027.41 of itemized deductions (mortgage interest $16,900 + SALT $14,127.41) that bring taxable income down to $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400, which cut $4,127.41 of deductible MA income and property tax. It then replaced its own $27,887 result with an unsupported $31,369." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductible state and local tax. It also used 2025 bracket thresholds. Finally it swapped its own $28,172 for an unexplained $31,379." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 property tax as SALT and left out the $8,484.41 MA income tax withholding estimate, so taxable income was overstated at about $151,544. It then replaced its own $28,968 bracket result with an unsupported $31,650." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,238.70 MA income tax liability in SALT instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. It also counted the $6 REIT income twice in AGI ($116 of dividends, AGI $174,094). Together these made taxable income about $252 too high and the tax about $60 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 property tax as SALT and left out the $8,484.41 MA income tax deduction allowed under the $40,400 cap. It also used 2025 bracket thresholds ($11,925/$48,475/$103,350) instead of the 2026 ones ($12,400/$50,400/$105,700)." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductions. It then gave $33,700 without a bracket calculation; the correct brackets applied to its own $147,194 taxable income give about $27,925." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated MA income tax at about $8,284 instead of PolicyEngine's withholding-based $8,484.41 (5% × (AGI − $4,400)). Its taxable income was about $200 too high ($143,266) and the tax about $50 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductible MA income and property tax. It also counted the $6 REIT income twice in AGI and skipped the QBI deduction." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and applied a $5,400 personal exemption with 10/15/25/28% brackets, but OBBBA made the TCJA rates permanent with no personal exemption. It also subtracted the $7,389 ESI premiums from the stated gross wages and left MA income tax out of SALT." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,312 personal exemption and pre-TCJA brackets, instead of OBBBA's permanent 2026 brackets with no exemption. It also understated SALT at $12,754.55 against the correct $14,127.41. Its submitted $27,594.14 does not even match the $29,794.41 its own reasoning reached." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It put MA income tax in SALT at $7,539.70 instead of PolicyEngine's withholding estimate of $8,484.41 (5% × (AGI − $4,400)), understating itemized deductions by about $945. It also counted the $6 REIT income twice in AGI, which pushed taxable income to $144,010.10." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules, a $5,050 personal exemption and 10/15/25/28% brackets, but under OBBBA 2026 keeps the 10/12/22/24% schedule with no personal exemption. This produced a different taxable-income base and a different rate schedule." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction. It never itemized the $16,900 mortgage interest plus $14,127.41 SALT (property tax and MA income tax), which totals $31,027.41 and is far above the standard deduction." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used TCJA-expired brackets and a personal exemption for 2026 instead of OBBBA's permanent 10/12/22/24% schedule. It also cut wages by the $7,389 ESI premiums, giving an AGI of $166,705 instead of $174,088.12." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired and applied a $5,300 personal exemption with pre-TCJA brackets instead of OBBBA's 2026 schedule. It also subtracted $7,389 of ESI premiums from the stated gross wages and left MA income tax out of SALT." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its stated taxable income of $169,008 leaves out nearly all of the $31,027.41 itemized deductions. Its $24,197 also does not follow from the 2026 brackets applied to that income (about $33,160), so both the deduction step and the rate step are wrong." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $7,389 ESI premiums from the stated gross wages, which PolicyEngine treats as the taxable wage base. It also left the $8,484.41 MA income tax out of SALT and counted only the $5,643 property tax. Its $27,858 does not match the 2026 brackets applied to its own $144,151 taxable income." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $24,907 matches about $134,600 of taxable income under the 2026 brackets, roughly $8,400 below the correct $143,059.47. That means it over-deducted beyond the $31,027.41 of itemized deductions (mortgage interest + property tax + $8,484.41 MA withholding estimate) and the $1.24 QBI deduction." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400, losing $4,127.41 of deductible MA income and property tax. Taxable income came out at $147,187 instead of $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset and applied a $5,150 personal exemption with 10/15/25/28% brackets, but OBBBA made the 10/12/22/24% schedule permanent with no exemption. Its SALT figure of $14,348 was close to the correct $14,127.41; the error came from the law regime." -us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used $8,388.70 for MA income tax instead of the $8,484.41 withholding estimate and counted the $6 REIT income twice in AGI. It also made an arithmetic error: $17,966 + 24% × $37,461.10 is $26,956.66, not the $26,916.66 it submitted." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the TCJA $10,000 instead of the 2026 OBBBA cap of $40,400. Even with that capped deduction, its $22,450 is about $5,500 below what the 2026 brackets give, so it misapplied the rate schedule as well." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,495 is higher than the $34,379 tax on the full $174,088 AGI with no deductions at all. So it applied none of the $31,027.41 itemized deductions and overstated the bracket tax as well." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a 2026 personal exemption that OBBBA permanently set to zero and applied the pre-TCJA rate schedule. That overstated the tax even though its itemized deductions of about $30,933 were close to the correct $31,027.41." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced wages by the $7,389 ESI premium, but the stated $175,002 gross wages is the federal wage base PolicyEngine taxes. This understated AGI and taxable income by several thousand dollars, so the tax came out about $2,000 too low." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted the Massachusetts income tax liability (about $8,238) as SALT instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. Itemized deductions came out about $246 short, and the tax $60 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It understated the MA income tax in SALT by about $140 compared with PolicyEngine's $8,484.41 withholding estimate (5% × (AGI − $4,400)). Taxable income was about $140 too high and the tax $34 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions uses about $8,230 of MA income tax (the liability) instead of the $8,484.41 withholding estimate (5% × (AGI − $4,400)). Taxable income came out at $143,313.70 instead of $143,059.47, and the tax $61 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted about $200 less MA income tax than PolicyEngine's $8,484.41 withholding estimate. Its taxable income of about $143,260 is above the correct $143,059.47, so the tax came out $48 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,330.10 of MA income tax instead of the $8,484.41 withholding estimate, which is 5% of AGI above the $4,400 exemption. Taxable income came out at $143,213.70 instead of $143,059.47, overstating the tax by $37." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions uses about $8,230 of MA income tax (the liability) instead of PolicyEngine's $8,484.41 withholding estimate (5% × (AGI − $4,400)). Taxable income came out at $143,313.70 and the tax $61 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $151,500 means it deducted only mortgage interest and property tax and left out the $8,484.41 MA income tax deduction allowed under the $40,400 OBBBA SALT cap." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset in 2026 and applied a $5,447 personal exemption and 10/15/25/28% brackets, but OBBBA made the 10/12/22/24% schedule permanent with zero personal exemption." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules, a $5,300 personal exemption and restored 10/15/25/28% brackets, instead of OBBBA's permanent 2026 schedule with no exemption. Its itemized total of $30,932 was nearly right, but the rate regime was wrong." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset, with a $5,350 personal exemption and 10/15/25/28% brackets, though OBBBA kept the 10/12/22/24% brackets and repealed the exemption permanently. The rate schedule pushed its tax $2,457 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA reverted brackets (10/15/25/28%) instead of OBBBA's permanent 2026 schedule. It also left the $8,484.41 MA income tax out of SALT, deducting only the $5,643 property tax." -us,scenario_081,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,335 of MA income tax instead of the $8,484.41 withholding estimate and counted the $6 REIT income twice in AGI. It also misapplied the brackets: its own $143,215 taxable income gives about $26,970 under the 2026 schedule, not $27,056." -us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 property tax as SALT and left out the $8,484.41 MA income tax withholding estimate that is deductible under the $40,400 OBBBA cap. Taxable income came out at $151,543.80 instead of $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It reported $0 tax with no derivation, even though taxable income of $143,059.47 at the 2026 single brackets produces $26,932.27 and no nonrefundable credits apply." -us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated the MA income tax in SALT at $8,538 against PolicyEngine's $8,484.41, which is 5% × (AGI $174,088.12 − $4,400). It also counted the $6 REIT income twice in AGI. Taxable income came out at about $143,012 instead of $143,059.47, so the tax was $11 too low." -us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly said $22,543 of itemized deductions was less than a $15,000 standard deduction, took the standard deduction, and left out the MA income tax from SALT. It also used outdated brackets, and its submitted $39,827.88 contradicts its own $31,281.32." -us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly concluded that $22,543 of itemized deductions was less than the standard deduction and took the standard deduction. It never itemized the $31,027.41 that includes the $8,484.41 MA income tax, so taxable income was overstated at $160,402." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It set Massachusetts income tax to 0 inside SALT and itemized only $22,543. PolicyEngine includes $8,484.41 of estimated MA withholding, for a total of $31,027.41. The model also miscounted the bracket base: $17,754.75 instead of $17,966 below $105,700. It then raised its own result to $29,385 for no stated reason." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It put about $8,230 of MA income tax (a liability estimate) into SALT. PolicyEngine uses 5% × (AGI − $4,400) = $8,484.41. The model also added the $6 REIT income on top of AGI, so taxable income was about $260 above the frozen withholding-based reference (~$143,320 vs $143,059.47); its $26,995 is $3.69 above the recorded liability-based reading of $26,991.31." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used the 2024 standard deduction ($14,600) and 2024 brackets. It deducted unreimbursed employee business expenses above the line, which is not allowed. It also wrongly concluded that $16,900 of mortgage interest falls short of the standard deduction, even though itemizing with uncapped-under-$40,400 SALT gives $31,027. It then replaced its own $30,957 with an arbitrary $29,825." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It gave a rough $33,000 with no deduction detail. Itemizing $31,027.41 (mortgage interest plus real estate tax plus $8,484.41 estimated MA tax) gives taxable income of $143,059.47 and tax of $26,932.27. The $33,000 figure matches taking only a standard deduction or a capped SALT deduction, with the bracket math overstated on top." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the pre-OBBBA $10,000 instead of the 2026 $40,400 cap, so itemized deductions were $26,900 rather than $31,027.41. It then threw out its own $27,887 bracket result and reported an unsupported $31,369." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired $10,000 SALT cap (2026 cap is $40,400), which reduced SALT by $4,127 relative to the frozen reference's calculation using an unlisted withholding estimate. It also used 2025 bracket thresholds instead of 2026 ones. It then replaced its computed $28,172 with an unexplained $31,379." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left Massachusetts income tax out of SALT and itemized only $22,543 instead of $31,027.41 including the $8,484.41 MA withholding estimate. That put taxable income about $8,500 above the frozen reference, which adds an unlisted withholding estimate; with no state income tax paid, the recorded sales-tax reading gives $28,763.81. It then inflated its own ~$28,968 to $31,650 with no basis." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used computed MA tax liability ($8,238.70) as the SALT income-tax component. PolicyEngine uses estimated MA withholding of 5% × ($174,088.12 − $4,400) = $8,484.41. Itemized deductions were $245.71 below the frozen reference, which uses an unlisted withholding estimate; its $26,992.66 is within $0.02 of the recorded corrected-liability reading of $26,992.64." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only real estate tax in SALT and omitted the $8,484.41 estimated MA income tax, so itemized deductions were $22,543 instead of $31,027.41. It also used 2025 bracket thresholds ($11,925/$48,475/$103,350) rather than 2026's ($12,400/$50,400/$105,700)." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the 2026 $40,400 cap, putting itemized deductions $4,127 below the frozen reference, which uses an unlisted withholding estimate. On its own $147,194 taxable income, 2026 brackets give about $27,925, so the $33,700 it reported is also a bracket-arithmetic error." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated MA income tax at about $8,284 for SALT instead of PolicyEngine's 5%-of-(AGI − $4,400) withholding estimate of $8,484.41. That cut itemized deductions by about $200 and put taxable income at ~$143,266 rather than $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the repealed $10,000 limit instead of the 2026 $40,400 cap, so it itemized $26,900, about $4,100 below the frozen reference's $31,027.41, which uses an unlisted withholding estimate." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA sunset and applied a $5,400 personal exemption with pre-TCJA 10/15/25/28% brackets. OBBBA made the TCJA rates and the zero exemption permanent. It also subtracted the $7,389 ESI premium from wages already reported as taxable and put no state income tax in SALT, while the frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law, with a $5,312 personal exemption and the 15/25/28% bracket schedule, even though 2026 keeps the TCJA brackets and has no exemption. Its reported $27,594.14 also contradicts the $29,794.41 it computed in its own reasoning." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It put $7,539.70 of state income tax into SALT instead of the $8,484.41 MA withholding estimate (5% × ($174,088.12 − $4,400)). It also counted the $6 REIT income in AGI, so taxable income was about $950 above the frozen withholding-based reference ($144,010 vs $143,059), and about $705 above the recorded liability-based reading ($143,305.47)." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset, subtracting a $5,050 personal exemption and using 15/25/28% brackets. Under 2026 law the exemption is $0 and the 10/12/22/24% brackets apply." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction even though mortgage interest plus real estate tax alone total $22,543. The frozen reference itemizes $31,027.41 using an unlisted MA withholding estimate; the recorded zero-state-income-tax reading itemizes $23,396. Its taxable income was about $16,000 above the frozen reference." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-expired brackets and a personal exemption, which 2026 law does not have. It also lowered AGI to $166,705 by subtracting the $7,389 ESI premium from wages that are already the taxable amount." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired and subtracted a $5,300 personal exemption with pre-TCJA brackets. It also reduced wages by the $7,389 ESI premium and itemized $22,543 with no state income tax in SALT; the frozen reference instead adds an $8,484.41 estimate of MA withholding that the prompt does not list." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $169,008 leaves out most itemized deductions: $31,027.41 in the frozen reference, using an unlisted withholding estimate, or $23,396 under the recorded zero-state-income-tax reading. The frozen reference's taxable income is $143,059.47. Its $24,197 tax also does not follow from 2026 single brackets applied to its $169,008 base, which would give about $33,160." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $7,389 ESI premium out of wages that are already the taxable amount. It also built SALT from real estate tax alone, omitting the $8,484.41 MA income tax estimate. Its $27,858 does not match 2026 brackets on its own $144,151 base either." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $24,907 implies taxable income about $8,400 below the frozen reference's $143,059.47, which uses an unlisted MA withholding estimate. That gap matches also excluding the $7,389 ESI premium from wages on top of the $31,027.41 itemized deductions." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated about $8,238 of MA income tax, close to the excluded frozen liability of $8,238.41; the recorded Part A loss-offset correction gives $8,232.90. It then capped SALT at the old $10,000 instead of the 2026 $40,400 cap, itemizing $26,900. The frozen federal reference itemizes $31,027.41 using an unlisted withholding estimate." +us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It treated pre-TCJA law as current for 2026, subtracting a $5,150 personal exemption and using 15/25/28% brackets. OBBBA kept the 10/12/22/24% schedule with no exemption, under which the frozen reference's calculation, using an unlisted withholding estimate, yields $26,932.27." +us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used $8,388.70 of MA tax in SALT instead of the $8,484.41 withholding estimate and added the $6 REIT income to AGI, which put taxable income about $100 above the frozen reference, whose withholding estimate is an unlisted input. It also summed its brackets wrong: $1,240 + $4,560 + $12,166 + $8,990.66 is $26,956.66, not $26,916.66." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the obsolete $10,000 instead of the 2026 $40,400 cap. Even on that capped base, 2026 brackets give about $27,900, so its $22,450 also understates the bracket computation." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,495 is about 24% of total income, which matches applying the top marginal rate to nearly all income instead of running the progressive 10/12/22/24% brackets on $143,059.47 of taxable income after $31,027.41 of itemized deductions." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added a personal exemption and used a pre-TCJA rate schedule for 2026. Its ~$30,933 itemized figure was close, but OBBBA set the exemption to $0 and kept the 10/12/22/24% brackets, which give the frozen reference's $26,932.27 when its unlisted withholding estimate is used." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced wages by the $7,389 ESI premium, even though the $175,002 wages are already the taxable amount. Taxable income came out about $8,400 too low, and tax was about $2,000 under the reference." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its answer is consistent with using its MA liability estimate of $8,238.70 in SALT; its federal explanation gives no state-tax figure. The frozen reference instead uses an unlisted withholding estimate of $8,484.41. Its $26,992.59 is within $0.05 of the recorded corrected-liability reading of $26,992.64." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It understated the MA income tax portion of SALT by about $140 compared with the $8,484.41 withholding estimate (5% of AGI above $4,400). That raised taxable income by the same amount and tax by $34." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions includes only $8,230.10 of MA income tax. PolicyEngine uses $8,484.41 (5% × ($174,088.12 − $4,400)), so the deduction was $254.31 below the frozen reference, which uses an unlisted withholding estimate; its $26,993.29 is within $1 of the recorded corrected-liability reading of $26,992.64." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $143,260 is roughly $200 above $143,059.47. The cause is an MA income tax figure in SALT that falls short of the $8,484.41 withholding estimate." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,330.10 of state income tax instead of the $8,484.41 MA withholding estimate (5% of AGI above the $4,400 exemption). Taxable income was $154 above the frozen withholding-based reference ($143,213.70), and its $26,969.29 was $37.02 above that reference but $22.02 below the recorded liability-based reading of $26,991.31." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $30,773.10 of itemized deductions embeds $8,230.10 of MA tax instead of the $8,484.41 withholding estimate (5% × ($174,088.12 − $4,400)). That put SALT $254 below the frozen reference, which uses an unlisted withholding estimate; its $26,993.29 is within $1 of the recorded corrected-liability reading of $26,992.64." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $151,500 matches itemizing only mortgage interest and real estate tax ($22,543). It left out the $8,484.41 estimated MA income tax that PolicyEngine includes in SALT under the $40,400 cap." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the TCJA sunset: a $5,447 personal exemption and restored 15/25/28% brackets. OBBBA made the TCJA rate schedule permanent with no exemption, so the 10/12/22/24% brackets apply; the $143,059.47 base belongs to the frozen reference's calculation using an unlisted withholding estimate." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had sunset and subtracted a $5,300 personal exemption with the restored 10/15/25/28% brackets. 2026 law keeps the TCJA brackets and a $0 exemption." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled 2026 as post-TCJA-sunset with a $5,350 personal exemption and 15/25/28% brackets. Its ~$30,828 itemized figure was close, but 2026 law applies the 10/12/22/24% brackets with no exemption." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA reverted brackets (15/25/28%) instead of the 2026 TCJA-permanent schedule. It also itemized $22,543 with no state income tax in SALT; the frozen reference instead adds an $8,484.41 estimate of MA withholding that the prompt does not list." +us,scenario_081,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It put about $8,335 of MA tax into SALT instead of the $8,484.41 withholding estimate. On its own ~$143,215 taxable income, the brackets give about $26,970, so its $27,056 adds an extra arithmetic overstatement." +us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $22,543 (mortgage interest plus real estate tax), giving taxable income of $151,543.80. The frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list, giving taxable income of $143,059.47; with no state income tax paid, the recorded SALT reading instead adds $853 of sales tax." +us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It reported $0 federal income tax with no reasoning. Even after $31,027.41 of itemized deductions and the QBI deduction, taxable income is $143,059.47, which produces $26,932.27 under 2026 single brackets." +us,scenario_081,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,384 of MA tax in SALT instead of the $8,484.41 withholding estimate and added $6 of REIT income to AGI, which put taxable income about $108 above the frozen reference, whose withholding estimate is an unlisted input. Its $27,047 also exceeds what 2026 brackets give on its own ~$143,167 base (about $26,992)." +us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It put $8,538 of MA income tax into SALT, above the $8,484.41 that 5% × ($174,088.12 − $4,400) gives. Itemized deductions were about $54 above the frozen reference, whose withholding estimate is an unlisted input; its tax was $11.27 below that reference and $70.31 below the recorded liability-based reading of $26,991.31." +us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the $15,000 standard deduction beat $22,543 of itemized deductions, which is backwards, and it also put no state income tax in SALT, while the frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list. It used pre-2026 brackets, and its reported $39,827.88 contradicts its own $31,281.32 computation." +us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction because it thought $22,543 of itemized deductions fell short. In fact $22,543 already exceeds the $16,100 standard deduction, and the frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list, bringing itemized deductions to $31,027.41." us,scenario_081,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It correctly worked out the PFML contribution as 0.46% × $175,002 = $805.01, then dropped it. Next it got the FICA-only total right at $13,387.65, but submitted $13,268.70, which matches none of its own components. The submitted value is an arithmetic and consistency error, and it also leaves out the PFML amount it had already calculated." -us,scenario_081,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It computed Social Security ($10,850.12) and Medicare ($2,537.53) correctly but left out the Massachusetts PFML employee contribution. The reference counts that contribution at 0.46% × $175,002 = $805.01. Its $13,387.65 is the answer under the reading that the employer absorbs the optional employee PFML share." -us,scenario_081,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It used the 2024 Social Security wage base of $168,600 as the 2026 cap, which cut Social Security tax to $10,453.20. The 2026 base is above $175,002, so the full wages are taxable ($10,850.12). It also said Massachusetts has no employee payroll tax and left out the $805.01 PFML contribution. Its final $13,383.41 doesn't match any of its own subtotals." -us,scenario_081,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It counted only federal FICA (6.2% + 1.45% on $175,002, rounded to $13,388) and left out the $805.01 Massachusetts PFML employee contribution at 0.46%. That matches the reading where the employer pays the employee's PFML share." -us,scenario_081,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It calculated FICA at $13,387.65 and then submitted $12,321.73, explaining it with a nonexistent cap even though it had just said all $175,002 of wages fall under the wage base. It also misread PFML as roughly $0 for the employee. The actual employee share is 0.18% family plus 40% of the 0.70% medical rate, which is 0.46%, or $805.01." -us,scenario_081,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added $10,850.12 + $2,537.53, which is $13,387.65, but submitted $12,717.62, an addition error of $670. It also set the Massachusetts PFML employee contribution to $0 instead of using the 0.46% employee rate ($805.01)." -us,scenario_081,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It treated the Massachusetts PFML employee contribution as about $0 and submitted FICA only. It also made a small slip, entering $13,385 instead of $13,387.65. The $805.01 gap to the reference is the PFML contribution at 0.46%, which it left out under the reading that the employer pays that share." -us,scenario_081,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It correctly calculated the Massachusetts PFML employee contribution as $805.01 at 0.46%, then chose to leave it out and submitted FICA only ($13,387.65). That is the reading where the employer pays the optional employee PFML share, while the reference includes it." -us,scenario_081,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It first reached the reference value of $14,192.66 using 0.46%, then switched to a made-up rate split: family 0.46%, medical 0.42%, with the employee paying all of medical plus 40% of family, for 0.604%. The actual 2026 rates are 0.18% family, which the employee can be charged in full, and 0.70% medical, of which the employee can be charged 40%, for a total employee share of 0.46%. Its PFML amount of $1,057.01 is $252 too high." -us,scenario_081,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It called Massachusetts PFML an employer-only charge and left out the $805.01 employee share at 0.46%. It then submitted $12,318.65 even though its own total was $13,387.65, an addition error of $1,069." -us,scenario_081,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly at $13,387.65 and explicitly left out the Massachusetts PFML employee contribution. The reference includes that contribution at 0.46% × $175,002 = $805.01, using the maximum share the employer can withhold." -us,scenario_081,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no employee state payroll tax and so missed the PFML employee contribution of 0.46% × $175,002 = $805.01. Its FICA-only $13,387.65 matches the reading where the employer absorbs the employee PFML share." -us,scenario_081,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It included Massachusetts PFML but used a 0.38% employee rate instead of 0.46%. The correct rate is the full 0.18% family rate plus 40% of the 0.70% medical rate, and the wrong rate understated PFML by $140 ($665 instead of $805.01)." -us,scenario_081,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It used an outdated PFML employee rate of 0.375% ($656.26) instead of 0.46% ($805.01). It also submitted $13,593.41, which doesn't match its own stated total of $14,043.91." -us,scenario_081,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It used a 0.44% PFML employee rate ($770.01) instead of 0.46%. The correct rate is 0.18% family plus 40% of the 0.70% medical rate, or $805.01, so it came out $35 short." -us,scenario_081,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It used the outdated 0.318% PFML employee rate ($556.51) instead of the 2026 employee share of 0.46%. The 2026 share is 0.18% family plus 40% of the 0.70% medical rate, which is $805.01." -us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"6.2% + 1.45% of $175,002 is $13,387.65, but it submitted $11,843, an arithmetic error of more than $1,500. It also left out the $805.01 Massachusetts PFML employee contribution at 0.46%." -us,scenario_081,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It subtracted the $7,389 employer-sponsored insurance premiums from the FICA wage base and taxed $167,613. The engine applies Social Security, Medicare, and PFML to the full $175,002 of gross wages. Its PFML add-on of about $580 is also below the correct $805.01 at 0.46%." -us,scenario_081,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It taxed $167,613 for FICA, which is wages minus the $7,389 ESI premiums, when the base should be the full $175,002. That understated Social Security by $458 and Medicare by $107. It also used a 0.24% PFML rate ($402) instead of 0.46% on $175,002 ($805.01)." -us,scenario_081,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It entered Social Security as $10,830 instead of 6.2% × $175,002 = $10,850.12. It also entered PFML as $765.01, about 0.437%, instead of 0.46% × $175,002 = $805.01. Together these undercount the total by about $60." -us,scenario_081,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It lowered the FICA base to $167,613 by subtracting the $7,389 employer-sponsored insurance premiums as pre-tax deductions, while the engine applies FICA to the full $175,002 of gross wages. It also left out the $805.01 Massachusetts PFML employee contribution." -us,scenario_081,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Its $13,444 total implies only about $56 of PFML on top of the $13,387.65 FICA. The correct employee PFML contribution is 0.46% × $175,002 = $805.01, so it badly understated the Massachusetts employee rate." -us,scenario_081,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare ($13,387.65) and left out the Massachusetts PFML employee contribution. The reference includes that contribution at the maximum withholdable 0.46% share, $805.01." -us,scenario_081,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"It built a PFML employee rate of 0.532% from a 0.352% medical share plus 0.18% family. The employee medical share is actually 40% of 0.70%, or 0.28%, so the correct total is 0.46% ($805.01), not $931." -us,scenario_081,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It said Massachusetts has no mandatory employee payroll tax and submitted FICA only ($13,387.65). That leaves out the PFML employee contribution of 0.46% × $175,002 = $805.01, which the reference includes as the maximum share the employer can withhold." -us,scenario_081,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It counted only federal FICA and left out the $805.01 Massachusetts PFML employee contribution at 0.46%. It also made a $1.72 arithmetic slip, giving $13,389.37 instead of $13,387.65." -us,scenario_081,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It wrongly said $175,002 of wages is above the $200,000 Additional Medicare Tax threshold. It then submitted $5,420, well below even the $10,850.12 Social Security component. The correct total is FICA of $13,387.65 plus $805.01 of PFML." -us,scenario_081,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It estimated PFML withholding at $962.51, about 0.55% of wages, instead of the 0.46% employee share. The correct share is 0.18% family plus 40% of the 0.70% medical rate, or $805.01, so PFML was overstated by $157.50." -us,scenario_081,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare (about $13,388) and left out the Massachusetts PFML employee contribution of 0.46% × $175,002 = $805.01 that the reference includes." -us,scenario_081,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied the combined 2.9% Medicare rate (employer plus employee) instead of the 1.45% employee rate: $175,002 × 9.1% ≈ $15,926. It also left out the $805.01 Massachusetts PFML employee contribution." -us,scenario_081,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It used the outdated 0.318% PFML employee rate ($556) instead of the 2026 employee share of 0.46%. That share is 0.18% family plus 40% of the 0.70% medical rate, or $805.01." -us,scenario_081,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to $175,002 and said there is no state payroll tax, which leaves out the Massachusetts PFML employee contribution of 0.46% ($805.01) that the reference includes." -us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax, so there was nothing to compare with the $14,192.66 reference." -us,scenario_081,payroll_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a 0.256% PFML employee rate ($448.01) instead of 0.46%. The correct rate is the full 0.18% family rate plus 40% of the 0.70% medical rate, which is $805.01." -us,scenario_081,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It submitted $0 even though the household has $175,002 of wages, so it skipped the $10,850.12 Social Security, the $2,537.53 Medicare, and the $805.01 Massachusetts PFML employee contribution." -us,scenario_081,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly at $13,387.65 but said Massachusetts has no mandatory employee payroll tax. That leaves out the PFML employee contribution of 0.46% × $175,002 = $805.01 that the reference includes." -us,scenario_081,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It calculated $13,387.65 for FICA, said Massachusetts has no employee payroll tax, and then submitted an unrelated $14,525.24 that it called a 'rounded' result. It also never included the $805.01 PFML employee contribution at 0.46%." -us,scenario_081,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It cited Additional Medicare Tax, which is $0 because wages are under $200,000, and never mentioned Massachusetts PFML. Its $13,922 is the $13,387.65 FICA plus an unexplained $534, not the 0.46% PFML contribution of $805.01." +us,scenario_081,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It correctly computed the MA PFML employee share as 0.46% × $175,002 = $805.01 but then dropped it, wrongly assuming state payroll tax is not modeled. It then submitted $13,268.70, a figure $118.95 below its own FICA sum of $13,387.65 and backed by no calculation." +us,scenario_081,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Its federal FICA was correct ($10,850.12 + $2,537.53), but it left out the Massachusetts PFML employee contribution entirely. The reference includes the maximum withholdable share of 0.46% × $175,002 = $805.01. The prompt's 'mandatory' wording does not settle whether the employer passes that share to the employee." +us,scenario_081,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It first used the stale $168,600 wage base, then went back to 7.65% but mis-multiplied it ($13,382.65 instead of $13,387.65), and submitted an unexplained $13,383.41. It said Massachusetts has no separate mandatory employee payroll tax and left out PFML; the reference assumes the maximum $805.01 deduction at 0.46%, but the employer may deduct less or none." +us,scenario_081,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Its answer is federal FICA only ($10,850 + $2,538, rounded). It left out the $805.01 Massachusetts PFML employee share (0.46% of wages), which the reference counts as withheld at the maximum permitted rate." +us,scenario_081,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It estimated PFML at about $0; the reference assumes the maximum deductible employee share of 0.46%, or $805.01, while the employer may deduct less or none. It then submitted $12,321.73, which contradicts its own subtotal of $13,387.65, with an incoherent wage-base excuse even though it had said the full wages were taxed." +us,scenario_081,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It wrote $10,850.12 + $2,537.53 but reported the sum as $12,717.62 instead of $13,387.65, a simple addition error. It also set the MA PFML contribution to zero, calling it unmodeled, when the 0.46% rate gives $805.01." +us,scenario_081,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It set the Massachusetts PFML employee contribution to ~$0, whereas the reference counts the maximum withholdable share of 0.46% × $175,002 = $805.01. It also submitted $13,385 against its own rounded FICA total of $13,388." +us,scenario_081,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It identified the exact MA PFML employee contribution ($805.01 at 0.46%) and then left it out, assuming it is not modeled. The reference adds it to FICA for a total of $14,192.66." +us,scenario_081,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It reached the frozen reference of $14,192.66 using the maximum deductible 0.46% share, then switched to a made-up 0.604% employee share by treating the 2026 split as 0.46% family / 0.42% medical, with employees paying all of medical and 40% of family. The reference uses published 2026 rates of 0.18% family and 0.70% medical; the employer may deduct up to all of the family portion plus 40% of the medical portion, or deduct less or none. The made-up rate inflated PFML to $1,057.01." +us,scenario_081,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It computed FICA as $13,387.65 twice, then submitted $12,318.65, which is a digit-transposition error. It also called MA PFML employer-side only and so left out the $805.01 employee share at 0.46%." +us,scenario_081,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It computed federal FICA correctly at $13,387.65 and explicitly left out the Massachusetts PFML contribution. The reference adds the maximum employee share of 0.46% × $175,002 = $805.01." +us,scenario_081,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It stated that Massachusetts has no employee state payroll tax, so it reported FICA only ($13,387.65). The reference includes the MA PFML employee contribution of 0.46% of wages ($805.01)." +us,scenario_081,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It used an outdated MA PFML employee rate of 0.38%, giving about $665, instead of the 2026 maximum employee share of 0.46% (0.18% family + 40% of 0.70% medical), which gives $805.01." +us,scenario_081,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It counted PFML at 0.375% ($656.26), below the reference's maximum deductible 0.46% share ($805.01); the employer may deduct less or none. It then submitted $13,593.41, which contradicts its own stated total of $14,043.91." +us,scenario_081,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It applied a 0.44% MA PFML employee rate ($770.01) instead of the 2026 maximum employee share of 0.46% ($805.01), which left the total $35 short." +us,scenario_081,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It used the old 2023 MA PFML employee rate of 0.318% ($556.51). The 2026 maximum employee share is 0.46% (0.18% family + 0.28% medical), which gives $805.01." +us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It said it applied 6.2% + 1.45% to $175,002, which equals $13,387.65, but reported $11,843, an arithmetic error. It also left out the $805.01 MA PFML employee contribution." +us,scenario_081,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It shrank the FICA wage base to $167,613 by subtracting the $7,389 employer-sponsored insurance premiums, even though nothing says they are pre-tax salary reductions. FICA and PFML both apply to the full $175,002. Its PFML estimate was also below the 0.46% rate." +us,scenario_081,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It cut FICA wages to $167,613 by treating the $7,389 ESI premiums as pre-tax, when the full $175,002 is taxable. It also used a 0.24% PFML rate instead of 0.46%." +us,scenario_081,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It understated Social Security as $10,830 instead of 6.2% × $175,002 = $10,850.12, and gave the PFML contribution as $765.01 instead of 0.46% × $175,002 = $805.01. The total came out about $60 short." +us,scenario_081,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It reduced FICA wages to $167,613 by subtracting the $7,389 ESI premiums, which are not given as pre-tax, instead of taxing the full $175,002. It also left out the $805.01 MA PFML employee contribution." +us,scenario_081,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It said it included MA PFML, but its $13,444 total is only about $56 above full-wage FICA of $13,387.65. The reference assumes the maximum deductible PFML share of 0.46% × $175,002 = $805.01; its implied PFML amount is below that maximum, but the employer may deduct less or none." +us,scenario_081,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It reported FICA only ($13,387.65) and left out the Massachusetts PFML employee share. The reference counts that share at the maximum 0.46% rate, or $805.01." +us,scenario_081,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"It used a 0.532% PFML employee rate built from a 0.352% medical employee share. The reference uses the maximum deductible medical share of 40% of 0.70% = 0.28%, plus 0.18% family, totaling 0.46% and $805.01; the model's $931 exceeds that maximum, and the employer may deduct less or none." +us,scenario_081,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It stated there is no Massachusetts mandatory employee payroll tax and reported FICA only. The reference adds the MA PFML employee share of 0.46% × $175,002 = $805.01." +us,scenario_081,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare, with a small $1.72 arithmetic drift above $13,387.65, and never considered the MA PFML employee contribution. The reference includes that contribution at the maximum 0.46% share, or $805.01." +us,scenario_081,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It wrongly said the $175,002 in wages exceeds the $200,000 Additional Medicare threshold, then reported $5,420, which is less than half of Social Security alone ($10,850.12). It also left out the $805.01 PFML contribution." +us,scenario_081,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Its FICA of $13,387.65 is correct, but it estimated PFML withholding at $962.51, an effective 0.55% rate. The 2026 maximum employee share is 0.46%, which gives $805.01." +us,scenario_081,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It reported federal FICA only (about $13,388) and left out the Massachusetts PFML employee contribution. The reference includes that contribution at 0.46% of wages ($805.01)." +us,scenario_081,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It applied the combined employer-plus-employee Medicare rate of 2.9% instead of the employee rate of 1.45%, giving 9.1% × $175,002 ≈ $15,926. It also left out the $805.01 MA PFML employee share." +us,scenario_081,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,It used the outdated 0.318% MA PFML employee rate ($556) instead of the 2026 maximum employee share of 0.46% ($805.01). +us,scenario_081,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It applied 7.65% to wages and stated there is no state payroll tax, so it left out the MA PFML employee contribution. The reference counts that contribution at the maximum 0.46% share, or $805.01." +us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_081,payroll_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a 0.256% PFML employee rate ($448.01) instead of the 2026 maximum employee share of 0.46% (0.18% family + 40% of 0.70% medical), which gives $805.01." +us,scenario_081,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It submitted $0 with no reasoning, ignoring payroll tax on $175,002 of wages entirely. The frozen reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 MA PFML, assuming maximum PFML withholding; counting only mandatory employee contributions gives federal tax alone, $13,387.65." +us,scenario_081,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"It estimated MA PFML at roughly $367, an effective rate of about 0.21%. The 2026 maximum employee share is 0.46%, which gives $805.01." +us,scenario_081,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It stated that Massachusetts has no modeled employee payroll tax and reported FICA only ($13,387.65). The reference adds the MA PFML employee contribution of 0.46% × $175,002 = $805.01." +us,scenario_081,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It computed FICA as $13,387.65 and said no mandatory state payroll taxes apply in Massachusetts, but then submitted an unexplained $14,525.24. It also never applied the 0.46% PFML employee rate, which gives $805.01." +us,scenario_081,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Its reasoning lists only Social Security, Medicare and Additional Medicare, which total $13,387.65 here, yet it submitted $13,922 with no calculation behind the extra $534. It also never applied the 0.46% MA PFML rate, which gives $805.01." us,scenario_081,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own working was effectively correct: $164,602 of Part B income at 5% is $8,230.10, plus $8.30 on interest and dividends, for about $8,238. It then raised the figure to $8,380 with a made-up 'rounding of components' adjustment and submitted a number its own work didn't support." -us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It got the wage-side deductions right ($2,000 FICA, $4,000 rent, $4,400 exemption). But it let the short-term capital loss cancel all the interest and dividends. The $56 of interest belongs in Part B, adding $2.80 of tax, and the dividends stay taxable in Part A, adding about $5.51; dropping both left it $8.41 short." -us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It made up $3,835 of nonrefundable 'property tax and personal exemption' credits. A single filer with no dependents who is under 65 and earns $175k gets none of them. It also never subtracted the $4,400 personal exemption, the $4,000 rent deduction or the $2,000 FICA deduction from Part B income." -us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It submitted a rounded $8,000 without computing Part B. The correct steps are $175,058 less the $2,000 FICA deduction, the $4,000 rent deduction and the $4,400 exemption, giving $164,658 at 5% plus about $5.51 of Part A dividend tax, for $8,238." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $11,287 of FICA, but Massachusetts caps that deduction at $2,000. It also skipped the renter deduction by claiming the head owns the home, even though $44,400 of rent is listed, which earns the $4,000 maximum. It then pushed its $7,966 result up to $8,730 with no basis." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 personal exemption. It left out the $2,000 Social Security/Medicare deduction and the $4,000 renter deduction, so its $8,538 is exactly $300 (5% of $6,000) too high." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It took the exemption and the $2,000 FICA deduction but never applied the $4,000 renter deduction, even with $44,400 of rent listed. It then added unexplained 'adjustments' that took its $8,438 up to $8,620." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at the pre-2023 $3,000 instead of the current $4,000. It also never took the $2,000 deduction for Social Security/Medicare withholding, which overstated taxable income by $3,000 ($150 of tax)." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached $8,539 using only the personal exemption, leaving out the $4,000 renter deduction and the $2,000 FICA deduction. It then cut the figure to $8,200 based on unexplained 'minor exemption differences' instead of applying those deductions." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It never took the $2,000 Part B deduction for Social Security/Medicare withholding. It also started from $174,088, which nets the $1,080 short-term capital loss against wage income; Massachusetts doesn't allow that, and the correct Part B base is $175,058." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used a $4,800 exemption instead of the $4,400 single exemption. It also never applied the $4,000 renter deduction or the $2,000 FICA deduction, and it started from federal AGI, which nets the short-term loss against wages." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $22,543 of federal-style itemized deductions (mortgage interest and real estate taxes), which Massachusetts doesn't allow. It also used a made-up $5,000 exemption and an understated $166,705 AGI, and it never applied the $4,000 rent or $2,000 FICA deductions." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It stacked $22,543 of federal itemized deductions (mortgage interest and property taxes) on top of the correct rent, FICA and exemption deductions. Massachusetts has no itemized deduction for those items. The $7,519.53 it submitted doesn't even match the $7,111.55 its own reasoning produced." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $16,900 of mortgage interest and $2,000 of real estate taxes, but Massachusetts Part B income allows neither. It also left out the $4,000 renter deduction and the $2,000 FICA deduction that do apply." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It capped the renter deduction at the old $3,000 instead of the $4,000 cap in effect since 2023. It also left out the $56 of interest in Part B and the Part A tax on the $110 of dividends." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to roughly $175,000 of gross income with no $4,400 personal exemption, no $4,000 renter deduction and no $2,000 FICA deduction. The correct Part B taxable income is $164,658." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,389 of employer-sponsored insurance premiums from the $175,002 of wages, but those wages are the full Massachusetts Part B wage base. That understated taxable income by about $7,400 even though the rent, FICA and exemption deductions were right." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 by subtracting the $7,389 of employer insurance premiums from wages, which shouldn't be reduced. It then left out the $4,000 renter deduction and the $2,000 FICA deduction, so the two mistakes partly offset each other." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It taxed roughly $175,000 at 5% with no personal exemption, renter deduction or FICA deduction. Those three subtractions total $10,400 and bring Part B taxable income down to $164,658." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from 'taxable wages' of $167,613 by subtracting the $7,389 of employer-sponsored insurance premiums from wages. The full $175,002 (plus $56 of interest) is the Part B base, so taxable income came out $7,445 too low." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,869 equals 5% of about $157,380. That is consistent with subtracting the $7,389 of employer-sponsored insurance premiums from wages before the otherwise-correct exemption, rent and FICA deductions, but Part B wages shouldn't be reduced by those premiums." -us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It never subtracted the $4,400 personal exemption or the $2,000 FICA deduction, and it used the outdated $3,000 rent cap instead of $4,000. It also netted the short-term capital loss against wage income, which Massachusetts Part B doesn't allow." -us,scenario_081,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-2023 $3,000 rent-deduction cap instead of $4,000. It also never took the $2,000 deduction for Social Security/Medicare withholding, which overstated taxable income by $3,000." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $6,440 implies about $128,800 of taxable income, roughly $36,000 below the correct $164,658 Part B plus about $110 Part A. That gap is consistent with subtracting federal itemized deductions (mortgage interest and property taxes), which Massachusetts doesn't allow." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Its $13,951 implies about $279,000 taxed at 5%, far above the household's $175,058 of Part B income. It didn't apply the 5% rate to $164,658, which is what's left after the $2,000 FICA, $4,000 rent and $4,400 exemption deductions." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It got the Part B wage deductions right but used the short-term capital loss to wipe out all $166 of interest and dividends. The $56 of interest is Part B income, adding $2.80 of tax, and the dividends stay taxable in Part A, adding about $5.51." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,915 implies about $158,300 of taxable income, roughly $6,500 below the correct $164,658. Since only the exemption and rent deduction were claimed, that figure requires an extra exclusion from wages, consistent with removing the $7,389 of employer insurance premiums." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It applied the exemption and the $4,000 renter deduction but left out the $2,000 Part B deduction for Social Security/Medicare withholding. That's 5% of $2,000, which accounts for the $100 overstatement." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $56 of interest as exempt and let the capital loss absorb the dividends. The interest is taxed in Part B ($2.80) and the dividends stay taxable in Part A (about $5.51), so it fell $8.31 short." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It applied only the personal exemption and the renter deduction, leaving out the $2,000 deduction for Social Security/Medicare withholding. It also netted the short-term loss into its income base, which Massachusetts Part B doesn't allow." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $4,400 exemption and $4,000 rent deduction from wages but not the $2,000 FICA deduction. It also dropped the taxable $56 of interest (Part B) and the $110 of dividends (Part A)." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the bank interest as exempt and let the capital loss absorb the dividends. The $56 of interest is taxed in Part B and the dividends in Part A, adding about $8.31 to its $8,230.10." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% straight to federal AGI of $174,094, skipping the $4,400 personal exemption, the $4,000 renter deduction and the $2,000 FICA deduction that bring Part B taxable income to $164,658." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $1,782 of unreimbursed employee business expenses, which Massachusetts Part B doesn't allow. It also left out the $4,000 renter deduction and the $2,000 FICA deduction that do apply." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It never applied the $4,000 renter deduction even though $44,400 of rent is listed. It also subtracted a made-up $1,000 capital-loss allowance from ordinary income." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $2,000 Part B deduction for Social Security/Medicare withholding. It also started from $174,094 of federal AGI, which nets the short-term capital loss against wages instead of using the $175,058 Part B base." -us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from AGI, leaving out the $4,000 renter deduction for the $44,400 of rent and the $2,000 FICA deduction." -us,scenario_081,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used the outdated $3,000 rent cap instead of $4,000 and left out the $2,000 Social Security/Medicare deduction. It also started from federal AGI, which nets the short-term loss against wages." -us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score." -us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It stated there was 'no separate rent expense input' despite $44,400 of listed pre-subsidy rent, so it left out the $4,000 renter deduction. It also netted the capital loss into the $174,088 base instead of using $175,058 of Part B income." -us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $0 with no reasoning, even though a single resident with $175,002 of wages owes Massachusetts' 5% tax on $164,658 of Part B income plus Part A tax on the dividends, $8,238.41 in total." -us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from wages, leaving out the $4,000 renter deduction and the $2,000 deduction for Social Security/Medicare withholding. That's a $6,000 overstatement of the base, or $300 of tax." -us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It wrongly said Massachusetts allows no relevant deductions and applied only the $4,400 exemption, leaving out the $4,000 renter deduction and the $2,000 FICA deduction. It also netted the short-term loss against wages." -us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,800 implies about $156,000 of taxable income, roughly $8,800 below the correct $164,658 Part B plus about $110 Part A. That reflects deductions beyond the $2,000 FICA, $4,000 rent and $4,400 exemption, which are the only Part B subtractions available." +us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own components came to about $8,238: $8,230.10 of Part B tax plus $8.30 on interest and dividends. It then added an unexplained ~$142 'rounding' adjustment and answered $8,380. It also left the $56 of interest out of Part B, but that shifted no tax because it taxed the interest at 5% elsewhere." +us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $1,080 short-term capital loss as wiping out the $166 of interest and dividends. Under the Massachusetts rules, the $56 of interest stays in Part B and the ~$110 of dividends is taxed at 5% in Part A. Leaving these out dropped about $8 of tax ($8,230 vs $8,238.41)." +us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted a made-up $4,550 of employee business expenses instead of the $4,400 exemption, $4,000 rent deduction, and $2,000 FICA deduction. It then subtracted an invented $3,835 of 'property tax and exemption' credits. No nonrefundable credit applies, and tax is 5% of $164,658 plus 5% of ~$110 of dividends." +us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded $8,000 instead of computing the base. That base is wages plus interest ($175,058), less the $2,000 FICA deduction, the $4,000 rent deduction, and the $4,400 exemption, which is $164,658 taxed at 5%, plus 5% on the dividends." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full ~$11,287 of FICA instead of the $2,000 Massachusetts cap. It also dropped the rent deduction because it assumed the person owns a home, though $44,400 of rent is listed and earns the $4,000 deduction. After all that it inflated its own $7,966 figure to $8,730 with no basis." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 personal exemption. It left out the $2,000 Social Security/Medicare deduction and the $4,000 rent deduction (50% of $44,400, capped). Those $6,000 of missing deductions overstate tax by $300." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It applied the exemption and the $2,000 FICA deduction but left out the $4,000 rent deduction for the listed $44,400 rent. It then raised its own $8,438 figure to $8,620 with unexplained 'small adjustments'." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at $3,000 instead of the current $4,000. It also left out the $2,000 Social Security/Medicare deduction, so taxable income came out $3,116 too high ($167,774 vs $164,658 plus $110 of dividends)." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It said the rent deduction did not apply because the person is an owner and never took the $2,000 FICA deduction, giving $8,539 after only the exemption. It then cut that to $8,200 with no rule behind the change. The correct path deducts $4,000 for rent and $2,000 for FICA." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the $4,000 rent deduction and the exemption but left out the $2,000 Social Security/Medicare deduction. It also netted the $1,080 short-term loss against ordinary income ($174,088 base), which Massachusetts Part B does not allow." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used a $4,800 exemption instead of the $4,400 single exemption. It started from federal AGI, which nets the capital loss, and it left out both the $4,000 rent deduction and the $2,000 FICA deduction." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $22,543 of federal-style itemized deductions (mortgage interest and real estate taxes), which Massachusetts does not allow, and used a $5,000 exemption instead of $4,400. It also started from an AGI that wrongly takes the $7,389 employer insurance premiums out of wages, and it left out the $4,000 rent and $2,000 FICA deductions." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It correctly took the $4,000 rent, $2,000 FICA, and $4,400 exemption deductions. It also deducted $22,543 of federal itemized deductions (mortgage interest and property taxes), which Massachusetts does not allow. Its submitted $7,519.53 does not match the $7,111.55 its own reasoning reached." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $16,900 of mortgage interest and $2,000 of real estate taxes, neither of which is a Massachusetts income tax deduction. It also left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at the outdated $3,000 instead of $4,000. It also left the $56 of interest out of Part B and never taxed the ~$110 of Part A dividends." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to roughly gross income ($175,000) with no deductions. It skipped the $4,400 personal exemption, the $4,000 rent deduction, and the $2,000 FICA deduction, which bring Part B taxable income down to $164,658." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 of 'taxable wages' by taking the $7,389 employer-sponsored insurance premiums out of wages. The listed $175,002 of gross wages is the Massachusetts Part B wage base. Its deductions ($4,400, $4,000, $2,000) were correct." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It took the $7,389 employer insurance premiums out of the $175,002 gross wages, a reduction that does not apply here. It also left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, applying only the $4,400 exemption." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Its $8,750 is 5% of about $175,000 of gross income with no deductions. It skipped the $4,400 exemption, the $4,000 rent deduction, and the $2,000 FICA deduction." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used $167,613 of 'taxable wages', which takes the $7,389 employer insurance premiums out of the $175,002 gross wages. Massachusetts Part B starts from the full $175,002 plus $56 of interest, so its base is understated by about $7,500." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,869 equals 5% of $157,379. That is wages net of the $7,389 employer insurance premiums ($167,613) plus $166 of investment income, minus the three deductions. It wrongly took the premiums out of the $175,002 gross wages." +us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $4,400 personal exemption and the $2,000 Social Security/Medicare deduction, and it capped the rent deduction at $3,000 instead of $4,000. It also netted the $1,080 short-term loss against ordinary income, which Part B does not allow." +us,scenario_081,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at the outdated $3,000 instead of $4,000 and left out the $2,000 Social Security/Medicare deduction. That left taxable income at $167,774 instead of $164,658 of Part B income plus $110 of dividends." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $6,440 implies about $128,800 of taxable income, roughly $36,000 below the correct $164,658 Part B base. That is consistent with subtracting federal-style itemized deductions (mortgage interest, property taxes), which Massachusetts does not allow, instead of only the $4,400 exemption, $4,000 rent deduction, and $2,000 FICA deduction." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Its $13,951 is about 8% of total income, well above Massachusetts' flat 5% rate. The correct result is 5% of $164,658 plus 5% of ~$110 of dividends, so it applied a wrong rate or a progressive rate schedule." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It let the $1,080 short-term capital loss absorb the $166 of interest and dividends. The $56 of interest belongs in Part B and the ~$110 of dividends is taxed at 5% in Part A, which adds about $8 (total $8,238.41)." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,915 is 5% of a federal AGI of $166,705 less the exemption and the $4,000 rent deduction. That AGI wrongly takes the $7,389 employer insurance premiums out of wages and nets the capital loss. It also left out the $2,000 Social Security/Medicare deduction." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It took the $4,400 exemption and the $4,000 rent deduction but left out the $2,000 Social Security/Medicare deduction. That overstates tax by about $100 ($166,774 base vs $164,658 plus dividends)." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It exempted the $56 of bank interest and let the short-term capital loss absorb the dividends. The interest is taxed in Part B and the ~$110 of dividends is taxed at 5% in Part A, which adds about $8." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $8,284 comes from a $174,088 base less the exemption and the $4,000 rent deduction. It left out the $2,000 Social Security/Medicare deduction and netted the $1,080 short-term loss against ordinary income." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $2,000 Social Security/Medicare deduction, which overstates tax by $100. It also let the capital loss erase the $56 of interest and ~$110 of dividends, which are taxed at 5%." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $56 of bank interest as exempt and let the capital loss absorb the dividends. The $56 is taxed in Part B and the ~$110 of dividends is taxed at 5% in Part A, which adds about $8." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to the full $174,094 AGI. It skipped the $4,400 personal exemption, the $4,000 rent deduction, and the $2,000 FICA deduction." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $1,782 of unreimbursed employee business expenses, which are not a Massachusetts Part B deduction here. It also left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied the exemption and the $2,000 FICA deduction but never took the $4,000 rent deduction (50% of the $44,400 rent, capped). That leaves taxable income at $167,774 instead of $164,658." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It took the exemption and the $4,000 rent deduction but left out the $2,000 Social Security/Medicare deduction. It also started from federal AGI, which nets the $1,080 short-term loss against Part B income." +us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from federal AGI. It left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction that reduce Part B income to $164,658." +us,scenario_081,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at the outdated $3,000 instead of $4,000 and left out the $2,000 Social Security/Medicare deduction." +us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It said there was 'no separate rent expense input' and ignored the listed $44,400 pre-subsidy rent. That rent earns the $4,000 Massachusetts rent deduction. It also netted the capital loss into a $174,088 base." +us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 even though a Massachusetts resident with $175,002 of wages owes the flat 5% tax. That tax is on $164,658 of Part B taxable income plus Part A dividends, for $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"It applied the exemption and the $2,000 payroll-tax deduction but left out the $4,000 rent deduction for the listed $44,400 rent. It also netted the capital loss against ordinary income." +us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from wages. It left out the $2,000 Social Security/Medicare deduction and the $4,000 rent deduction, which overstates Part B tax by $300." +us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from federal AGI. It left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it let the short-term loss reduce ordinary income." +us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,800 implies $156,000 of taxable income, about $8,700 below the correct $164,658 Part B base plus $110 of dividends. That means it subtracted deductions beyond the $4,400 exemption, $4,000 rent deduction, and $2,000 FICA deduction that Massachusetts allows." us,scenario_081,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_chip_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 1 as within the applicable CHIP age range. The trace applies CHIP's age criterion against the child and returns false, while Medicaid eligibility also returns NONE." us,scenario_082,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested child1_chip_eligible output and supplied no explanation, violating the required submission contract." @@ -6870,52 +6955,53 @@ us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,health_coverage,Fa us,scenario_082,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the WIC income requirement. It failed to compare approximately $90,132 of annual household income with the much lower 185%-of-poverty limit for a two-person household." us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age under five as sufficient for WIC eligibility. Age establishes categorical eligibility only; the household must also pass the 185%-of-poverty income test, which it fails at approximately $90,132 of annual income." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work ends at about $10,353, but it submitted $14,748, which none of its steps support. That work also left out the $700 non-itemizer charitable deduction and the $68 SALT refund, and it used a $2,000 CTC instead of the 2026 $2,200 and wrong HoH brackets (12% band ending at $65,900 instead of $67,450)." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The structure was right, but it computed $10,444.60 of ordinary tax on $79,380. The 2026 HoH brackets (10% to $17,700, 12% to $67,450, 22% above) give $10,364.60, so its figure is $80 too high. It also deducted the $8 traditional IRA contribution, which is fully phased out for an active 401(k) participant at about $117.6k AGI." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It used the single-filer $14,600 standard deduction instead of the $24,150 HoH amount. It also netted the entire $33,000 long-term loss against income instead of capping the net capital loss deduction at $3,000. Its final $23,437 then contradicts its own roughly $8,506 pre-credit tax with no derivation." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rounded guess that mentions only the CDCC and never subtracts the $2,200 nonrefundable CTC. The $14,500 is above even the correct pre-credit tax of $12,379.77, so it also overstated regular tax: it never applied the $3,000 capital-loss limit, the 15% qualified-dividend rate, or the $24,850 of standard plus charitable deductions." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $23,500 HoH standard deduction instead of $24,150 and pre-OBBBA brackets (10% to $17,000, 12% to $64,850) instead of $17,700/$67,450. It also left out the $700 non-itemizer charitable deduction and the $68 SALT refund income, so it overstated ordinary tax." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $23,000 HoH standard deduction, 2025-style brackets (12% band ending at $64,850), and a $2,000 CTC instead of the 2026 $2,200, and it left out the $700 non-itemizer charitable deduction. That gave $10,460, which it then raised to $11,150 without any basis." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It guessed a $24,500 standard deduction instead of $24,150, left out the $700 non-itemizer charitable deduction, and taxed ordinary income with 10% to $17,000 and 12% to $64,850 instead of the 2026 HoH thresholds of $17,700 and $67,450. That inflated its ordinary tax to $10,716 versus about $10,366." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $8 traditional IRA contribution. The head is an active participant in an employer 401(k), and AGI of about $117.6k is above the HoH deduction phase-out range, so the IRA deduction is zero. That $8 at 22% is the whole shortfall." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It counted the $2,289 short-term gain as separate ordinary income even though it nets into the $30,711 capital loss. It left the $13,424 of qualified dividends out of AGI and then subtracted them again from taxable income, so they were excluded twice. It also used a $22,500 standard deduction, a $2,000 CTC, and pre-OBBBA brackets." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own steps produced $10,739, but it submitted $13,920 with no supporting computation. Those steps also used a $22,500 standard deduction, 2024-level brackets (10% to $16,550, 12% to $63,100), a $2,000 CTC, and no $700 non-itemizer charitable deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $24,150 standard deduction, the brackets, and both credits right. It left out the $700 charitable deduction that non-itemizers can take starting in 2026 (up to $1,000 for HoH), so taxable income was $700 too high and tax was overstated by $154." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted both the full $33,000 long-term loss and a separate $3,000 capital-loss deduction instead of capping the net loss deduction at $3,000. It invented a $46,000 HoH standard deduction. It also applied a 35% CDCC rate on a $5,000 expense cap (correct is 20% of $3,000) and a $300 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: $9,680 of personal exemptions, 10/15/25% brackets, and a $1,000 CTC. The One Big Beautiful Bill Act made the TCJA structure permanent, so 2026 has no personal exemptions, a $24,150 HoH standard deduction, a $2,200 CTC, and 10/12/22% brackets." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired: $11,000 of personal exemptions, pre-2018 brackets, and a $1,000 CTC phased out above the old $75,000 HoH threshold. Under OBBBA in 2026 there are no exemptions, the HoH standard deduction is $24,150, and the $2,200 CTC phases out only above $200,000." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $23,100 HoH standard deduction instead of $24,150, a $2,300 CTC instead of $2,200, and brackets that taxed $81,130 of ordinary income at $11,059.60 instead of about $10,750 under the 2026 thresholds. It also deducted the $8 IRA contribution, which is phased out, and left out the $700 non-itemizer charitable deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA law: uncapped SALT itemizing of $18,564, $10,000 of personal exemptions, and a $1,000 CTC. Under 2026 OBBBA law there are no personal exemptions, the CTC is $2,200, and the $24,150 standard deduction plus the $700 non-itemizer charitable deduction beats itemizing." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $100,571 AGI is about $17,090 below the correct $117,661 because it did not include all $20,480 of dividends and limit the capital loss to $3,000. It also never subtracted the $600 CDCC and used a $2,000 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the $21,208 of employer-sponsored insurance premiums from wages, giving $96,446 AGI, and applied pre-TCJA personal exemptions of $10,400 with no CTC. In 2026 there are no personal exemptions, and the $2,200 CTC applies in full because AGI is far below the $200,000 phase-out." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reduced AGI to $96,454 by subtracting the employer-sponsored insurance premiums, which are not part of the input wages. It also applied TCJA-sunset personal exemptions and a $1,000 CTC instead of the 2026 OBBBA rules: no exemptions, the $24,150 standard deduction plus $700 charitable deduction, and a $2,200 CTC." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. $12,790.25 is roughly the correct pre-credit tax of $12,379.77, which fits a calculation that never subtracted the $2,800 of nonrefundable credits (the $2,200 CTC and $600 CDCC) that bring the tax to $9,579.77." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It explicitly subtracted personal exemptions. Those are permanently zero under OBBBA, so its taxable income was too low and its tax understated by about $2,200. The 2026 HoH return gets only the $24,150 standard deduction and the $700 non-itemizer charitable deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,384 fits netting the entire $30,711 capital loss against income. That drops AGI to about $90k and leaves the qualified dividends in the 0% bracket. The capital loss deduction is capped at $3,000, so AGI is $117,661 and all $13,424 of dividends are taxed at 15%." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It subtracted a $2,000 CTC instead of the 2026 $2,200 and left out the $700 non-itemizer charitable deduction. It also overstated pre-credit tax by about $320 beyond those two items by using outdated HoH bracket or standard deduction amounts, ending $675 high." -us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the HoH standard deduction at $23,063 instead of $24,150 and the brackets at 10% to $16,913 and 12% to $64,575 instead of $17,700 and $67,450. It left out the $700 non-itemizer charitable deduction and the SALT refund income, and it used a $2,000 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $21,208 of employer-sponsored insurance premiums as deductible medical expenses and itemized $23,266. Without the premiums, medical costs fall below the 7.5% floor, and even its itemized figure is below the $24,850 available to a non-itemizer ($24,150 standard plus $700 charitable). It also overstated ordinary tax with wrong bracket thresholds." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated capital gains as positive taxable income instead of recognizing a $30,711 net capital loss limited to a $3,000 deduction. Its unsupported $12,719 is $3,139 above the correct figure, more than the $2,800 of CTC and CDCC it says it subtracted." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized real estate taxes, medical, and charity, but medical costs are below the 7.5% AGI floor and the itemized total is below the $24,850 of standard plus charitable deductions available to a non-itemizer. Its $17,303 is above even the correct pre-credit tax of $12,379.77, so it also skipped the $3,000 capital-loss limit, the 15% qualified-dividend rate, or the $2,800 of nonrefundable credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $24,415, which is less than the $24,850 a non-itemizer gets ($24,150 HoH standard deduction plus the $700 charitable deduction for non-itemizers). It also subtracted a $2,000 CTC instead of the 2026 $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its AGI of about $89,951 nets the entire $30,711 capital loss against income. The net capital loss deduction is capped at $3,000, so AGI is $117,661, and the qualified dividends are taxed at 15% rather than falling into the 0% bracket." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It got AGI, the standard deduction, the brackets, and both credits right. It left out the 2026 non-itemizer charitable deduction ($700 of cash gifts, up to $1,000 for HoH), so taxable income was $700 too high and tax was overstated by $154." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the right $2,200 CTC and $600 CDCC but left out the $700 non-itemizer charitable deduction. It also overstated pre-credit tax by about $500 more through understated HoH bracket thresholds or standard deduction, implying about $13,036 of pre-credit tax versus $12,379.77." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized medical expenses, which only exceed the 7.5% AGI floor if the $21,208 of employer-sponsored insurance premiums are counted, and those premiums are not deductible. Real estate taxes, NY income tax, and charity alone come to less than the $24,850 non-itemizer total, so its deduction was too large and its tax understated." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the $24,150 standard deduction, the 2026 brackets, and the $2,200 CTC and $600 CDCC correctly. It left out the $700 charitable deduction for non-itemizers that applies starting in 2026, overstating tax by $154." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $93,504 taxable income leaves out the $700 non-itemizer charitable deduction. It also includes an $8 traditional IRA deduction that is fully phased out for an active 401(k) participant at this AGI. The missing $700 deduction accounts for the $152 overstatement after the $8 offset." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized medical expenses, which requires counting the non-deductible $21,208 of employer-sponsored insurance premiums. That pushed its itemized total above the correct deduction of $24,850 ($24,150 standard plus the $700 non-itemizer charitable deduction) and understated tax by about $778." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no computation. $14,000 is above even the correct pre-credit tax of $12,379.77, so it never subtracted the $2,200 CTC and $600 CDCC and also overstated regular tax by ignoring the $3,000 capital-loss deduction and the $24,850 of standard plus charitable deductions." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026: two personal exemptions and a CTC fully phased out. OBBBA made the TCJA structure permanent, so there are no exemptions and the $2,200 CTC applies in full. It also wrongly subtracted the $21,208 of employer-sponsored insurance premiums from wages, giving $96,454 AGI." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: two $5,300 exemptions, 10/15/25% HoH brackets, and a $1,000 CTC phased out above $75,000. In 2026 under OBBBA there are no exemptions, brackets are 10/12/22%, the $24,150 standard deduction plus $700 charitable deduction applies, and the $2,200 CTC phases out only above $200,000." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its $28,094 of itemized deductions includes medical expenses above the 7.5% floor, which only works by counting the non-deductible $21,208 of employer-sponsored insurance premiums. Without them, itemizing falls below the $24,850 of standard plus non-itemizer charitable deductions, so it understated taxable income." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules: a $12,591 HoH standard deduction, pre-2018 brackets, and no CTC. Under 2026 OBBBA law the HoH standard deduction is $24,150, the $700 non-itemizer charitable deduction applies, and the $2,200 CTC is subtracted in full." -us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It left out the $700 non-itemizer charitable deduction and deducted the phased-out $8 IRA contribution, giving $93,504 of taxable income instead of $92,811. It then taxed $80,080 of ordinary income at about $10,638 instead of $10,519 under the 2026 HoH brackets (12% to $67,450)." -us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score." -us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the capital loss as wiping out the tax. The net capital loss deduction is capped at $3,000, leaving $92,811 of taxable income and $12,379.77 of pre-credit tax, which the $2,800 of nonrefundable credits cannot eliminate." -us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI, $24,150 standard deduction, brackets, and both credits are right. It left out the $700 charitable deduction for non-itemizers that applies starting in 2026, so taxable income was $93,512 instead of $92,811 and tax was overstated by $154." -us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the full $30,711 capital loss against income instead of capping the deduction at $3,000. It used a $21,900 standard deduction and treated only $400 of the CTC as nonrefundable, reaching $6,509, and then submitted an unsupported $12,866.90." -us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It used the single-filer $15,750 standard deduction instead of the $24,150 head-of-household amount. It invented a $6,237 QBI deduction even though there is no business income, and it omitted the $600 CDCC while using a $1,875 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It submitted $14,748, which matches neither of its own figures: the ~$10,353 final it computed or the ~$12,953 pre-credit tax. The derivation itself also missed the $700 non-itemizer charitable deduction, wrongly deducted the $8 IRA contribution (phased out for a 401(k) participant), and used a $2,000 CTC instead of the 2026 $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its deductions are essentially right, but it put the tax on $79,380 of ordinary income at $10,444.60. The 2026 head-of-household brackets (10% to $17,700, 12% to $67,450, 22% above) give $10,364.60, so its bracket thresholds overstated tax by $80. It also wrongly took the $8 IRA deduction, which is phased out for an active 401(k) participant at this AGI." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It filed as single with a $14,600 standard deduction instead of head of household with $24,150 plus the $700 non-itemizer charitable deduction, and it took the CDCC as 20% of the full $9,600 instead of the $3,000 one-child cap. It then dropped its own ~$8,506 computation and asserted $23,437 with no supporting arithmetic." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no computation. Its $14,500 is above the correct pre-credit tax of $12,379.77, so it never correctly applied the $24,150 head-of-household standard deduction plus the $700 non-itemizer charitable deduction, the $3,000 capital-loss deduction, or the $2,200 CTC and $600 CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $23,500 standard deduction instead of $24,150, left out the $700 non-itemizer charitable deduction, and applied stale head-of-household brackets (12% ending at $64,850 instead of $67,450). That overstated both taxable income ($94,094 vs $92,811) and the ordinary-rate tax." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $23,000 standard deduction with outdated brackets (12% ending at $64,850), skipped the $700 non-itemizer charitable deduction, and took a $2,000 CTC instead of $2,200. It then raised its own $10,460 result to $11,150 with no stated basis." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $24,500 standard deduction instead of the 2026 head-of-household $24,150, left out the $700 non-itemizer charitable deduction, and taxed ordinary income with stale bracket thresholds ($17,000/$64,850 instead of $17,700/$67,450). The result was $9,930 instead of $9,579.77." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It explicitly assumed no active-participant phase-out and deducted the $8 traditional IRA contribution. Because Head contributes to a 401(k), that deduction is fully phased out at $117k head-of-household AGI. Together with rounding the SALT refund and wage figures, this left taxable income $7.12 too low ($92,804 vs $92,811.12), or $1.57 of tax at 22%." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left the $13,424 of qualified dividends out of AGI ($106,519) and then subtracted them again from taxable income, so they were removed from the ordinary base twice. It also double-counted the short-term gain inside the capital-loss netting and used a $22,500 standard deduction and a $2,000 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $10,739 and then submitted an unrelated $13,920. The derivation itself used a $22,500 standard deduction, stale brackets (12% ending at $63,100), no $700 non-itemizer charitable deduction, and a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its AGI, $24,150 standard deduction, brackets and credits are all correct, but it left out the 2026 above-the-line charitable deduction for non-itemizers, which covers the full $700 gift. That leaves taxable income $700 too high and overstates tax by $154 at 22%." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the full $33,000 long-term loss and then another $3,000 instead of capping the net loss deduction at $3,000. It also used a $46,000 standard deduction, roughly double the head-of-household $24,150, and took the CDCC as 35% of $5,000 instead of 20% of the $3,000 one-child cap." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA rules for 2026: $9,680 of personal exemptions, 10/15/25% brackets, and a $1,000 CTC. OBBBA made the TCJA structure permanent, so the correct inputs are no exemptions, the $24,150 standard deduction plus the $700 charitable deduction, the 10/12/22% brackets, and a $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, claiming $11,000 of personal exemptions and a $1,000 CTC that it then phased out entirely. Under 2026 OBBBA law there are no exemptions, the head-of-household standard deduction is $24,150, and the full $2,200 CTC applies at this income." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $23,100 standard deduction, left out the $700 non-itemizer charitable deduction, and overstated the tax on $81,130 of ordinary income ($11,059.60 vs $10,749.60 under the 2026 head-of-household brackets). It then used a $2,300 CTC instead of the $2,200 statutory amount." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules, with $10,000 of personal exemptions and a $1,000 CTC, and itemized an inflated $18,564 of SALT. The correct path is the $24,150 head-of-household standard deduction plus the $700 non-itemizer charitable deduction, no exemptions, and a $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It understated AGI as $100,571 instead of $117,661.12, so most of the dividend income dropped out. It also subtracted only a $2,000 CTC, leaving out the $600 CDCC and $200 of the 2026 $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 of employer-sponsored insurance premiums from wages, giving an AGI of $96,446 instead of $117,661. It also applied pre-TCJA personal exemptions ($10,400) and gave no child tax credit at all, subtracting only the $600 CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut AGI to $96,454 by excluding the $21,208 of employer-sponsored insurance premiums from wages. It also added TCJA-sunset personal exemptions and used a $1,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. Its $12,790.25 is above the correct pre-credit regular tax of $12,379.77, so it never netted the $2,200 CTC and $600 CDCC against a correctly computed head-of-household liability." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It deducted personal exemptions, which do not exist under 2026 OBBBA law, and so understated taxable income. It also named only the CDCC as a nonrefundable credit and did not properly apply the $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $3,384 implies taxable income roughly $28k too low, which is what you get by netting the full $30,711 capital loss against income instead of capping the deduction at $3,000." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $2,000 CTC instead of the 2026 $2,200 and left out the $700 non-itemizer charitable deduction, which together overstate tax by $354. Its ordinary-rate tax was also computed with stale head-of-household brackets." +us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used an estimated $23,063 standard deduction and brackets (10% to $16,913, 12% to $64,575) instead of the 2026 values of $24,150, $17,700 and $67,450. It also left out the $700 non-itemizer charitable deduction and the $67.50 SALT refund income, and used a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $21,208 of employer-sponsored insurance premiums as medical expenses and itemized $23,266. That total is still below the $24,150 standard deduction plus the $700 non-itemizer charitable deduction, so itemizing raised taxable income. It also overstated the ordinary-rate tax using stale brackets ($11,046 vs $10,715 on $80,972)." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It skipped the computation. Its $12,719 is above the correct pre-credit regular tax of $12,379.77, so it never correctly applied the $24,850 of head-of-household deductions or netted the $2,800 of CTC and CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized real-estate tax, medical and charity instead of taking the larger $24,150 head-of-household standard deduction plus the $700 non-itemizer charitable deduction. Its $17,303 is above even the correct pre-credit tax of $12,379.77, so it also failed to apply the $2,200 CTC and $600 CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It took $24,415 of itemized deductions, less than the $24,850 available from the $24,150 standard deduction plus the $700 non-itemizer charitable deduction. It also subtracted a $2,000 CTC instead of the 2026 $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $89,951 AGI comes from netting the entire $30,711 capital loss against income. The net capital loss deduction is capped at $3,000, so the correct AGI is $117,661.12 and $27,711 carries forward." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its AGI, $24,150 standard deduction and credits are correct, but it left out the 2026 charitable deduction for non-itemizers, which covers the full $700 gift. That leaves taxable income $700 too high and overstates tax by $154." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the standard deduction, the $2,200 CTC and the $600 CDCC but not the $700 non-itemizer charitable deduction, and its regular tax ran about $500 over the $12,379.77 the 2026 head-of-household brackets produce. The result was $10,236 instead of $9,579.77." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized medical expenses that included the $21,208 of employer-sponsored insurance premiums. Those premiums are not a deductible medical cost here, so itemized deductions came out at about $28.5k, above the correct $24,150 standard deduction plus the $700 charitable deduction, and taxable income was understated." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard deduction, the $3,000 capital-loss cap and the $2,200 and $600 credits correctly, but left out the 2026 non-itemizer charitable deduction of $700. That overstates tax by $154." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $93,504 leaves out the $700 non-itemizer charitable deduction. It also includes the $8 IRA deduction, which is phased out for a 401(k) participant at this AGI. The correct taxable income is $92,811.12." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized medical costs that counted the $21,208 of employer-sponsored insurance premiums. Without those premiums the itemized total falls below the $24,150 standard deduction plus the $700 charitable deduction, so its larger deduction understated taxable income and tax." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no computation. Its $14,000 is above the correct pre-credit regular tax of $12,379.77, so it never applied the $24,150 head-of-household standard deduction plus the $700 charitable deduction, the $3,000 capital-loss deduction, or the $2,800 of CTC and CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset, claimed two personal exemptions and phased the CTC out entirely. It also subtracted the $21,208 of employer-sponsored insurance premiums from wages. Under 2026 OBBBA law AGI is $117,661, there are no exemptions, and the full $2,200 CTC applies." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-sunset rules: 10/15/25% brackets, two $5,300 exemptions, and a $1,000 CTC phased out above $75,000. Under 2026 law the brackets are 10/12/22%, there are no exemptions, the $24,150 standard deduction plus the $700 charitable deduction applies, and the full $2,200 CTC is allowed." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its $28,094 of itemized deductions treats the $21,208 of employer-sponsored insurance premiums as medical expenses above the 7.5% floor. Those premiums are not deductible here, so the correct deduction is the $24,150 standard deduction plus the $700 non-itemizer charitable deduction." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used post-TCJA-expiration parameters, with a $12,591 standard deduction and higher bracket rates, and gave no child tax credit at all. Under 2026 OBBBA law the head-of-household standard deduction is $24,150, there is a $700 charitable deduction, and the $2,200 CTC applies." +us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It left out the $700 non-itemizer charitable deduction and took the phased-out $8 IRA deduction. It also overstated ordinary tax on $80,080 ($10,638 vs $10,519 under the 2026 head-of-household brackets of 10% to $17,700 and 12% to $67,450)." +us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the $30,711 net capital loss as offsetting most of the income and concluded tax was $0. The loss deduction against ordinary income is capped at $3,000, which leaves $92,811 of taxable income and $12,379.77 of tax before the $2,800 of credits." +us,scenario_082,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 of employer-sponsored insurance premiums as a pre-tax reduction of wages, cutting taxable income to about $71,600. With wages counted at $100,013.62 net of only the 401(k) deferral, taxable income is $92,811.12." +us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI, $24,150 standard deduction, brackets and credits are correct, but it left out the 2026 charitable deduction for non-itemizers, which covers the full $700 gift. Taxable income is therefore $700 too high and tax $154 too high." +us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the full $30,711 capital loss into AGI instead of capping the deduction at $3,000, and treated all but $400 of the CTC as refundable when the full $2,200 is usable against this liability. It then dropped its own ~$6,509 result and submitted an unrelated $12,866.90." +us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It filed as single with a $15,750 standard deduction instead of head of household with $24,150 plus the $700 charitable deduction. It also invented a QBI deduction with no business income, started from an inflated $139,775 pre-adjustment AGI, used a $1,875 CTC instead of $2,200, and dropped the $600 CDCC." us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied only the ACTC limitation formula — lesser of $1,700 and 15% of earned income over $2,500 — and never performed the ordering step in which the nonrefundable CTC is first applied against tax liability and only the unused remainder becomes refundable. Its own reasoning conceded the head 'has sufficient nonrefundable CTC capacity' and then discarded that conclusion; with roughly $117,650 of AGI the pre-credit federal tax absorbs the entire credit, leaving $0. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200 amount." us,scenario_082,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the $1,700 statutory refundable ceiling as the answer after checking only the $2,500 earned-income floor, skipping the requirement that the refundable ACTC equals the CTC remaining after the nonrefundable portion offsets tax liability. Even at its own stated AGI of about $86,949, head-of-household tax before credits far exceeds the $2,200 CTC, so the credit is fully used nonrefundably and refundable credits are $0." us,scenario_082,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It quoted $1,000, the pre-TCJA additional child tax credit cap, which has no basis in 2026 law where the credit is $2,200 per child with a $1,700 refundable ceiling. The controlling error is treating a refundable amount as automatic: this filer's federal tax before credits exceeds the full $2,200 credit, so the nonrefundable CTC absorbs all of it and $0 is refundable." @@ -6931,145 +7017,148 @@ us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_l us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied NYC head-of-household brackets to an NYC taxable income of $79,254, treating the household as NYC residents when only state = NY was given. Because NYC residence is an unlisted status that defaults to false, no NYC tax arises and local income tax is $0." us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"It said outright that it 'assumed to reside in NYC (the only NY locality with a local income tax),' which breaks the instruction to treat unlisted facts as false. Without NYC residence, its whole NYC bracket, school tax credit and child care credit computation does not apply, and local income tax is $0." us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"It stated that 'NYC resident income tax applies' to NY taxable income of about $104,454 and applied NYC brackets, but the household is never placed in NYC. Under the default-false rule for unlisted status facts, the household owes no NYC tax and local income tax is $0." -us,scenario_082,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92 but deliberately left out NY PFL ($411.91) and DBL ($31.20). It read 'mandatory employee state payroll taxes' as excluding contributions that NY employers are allowed but not required to withhold. The reference counts the statutory maximum employee contributions, which add $443.11." -us,scenario_082,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($6,212.09 + $1,452.83 = $7,664.92). It left out the NY employee PFL contribution (0.432% of wages, capped at $411.91) and the DBL contribution ($31.20), which the reference counts at their maximums as mandatory employee state payroll taxes." -us,scenario_082,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Applied a made-up 0.62% PFL rate with no cap ($621.21), then padded the total with unnamed 'other NY withholdings' to reach $8,304.92. In 2026 the NY PFL employee rate is 0.432% with a $411.91 annual maximum, and DBL is $31.20, so NY adds $443.11." -us,scenario_082,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Reported rounded FICA only ($7,665). It left out NY PFL ($411.91, 0.432% capped) and DBL ($31.20), reading the employer-optional NY employee contributions as not mandatory, while the reference includes them at the statutory maximum." -us,scenario_082,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Got FICA and DBL ($31.20) right, but after citing the 2025 0.388% PFL rate it fell back to an unexplained $49.31 PFL figure. The 2026 PFL is 0.432% of wages capped at $411.91, so NY adds $443.11, not $80.51." -us,scenario_082,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Said NY has no mandatory employee payroll tax and reported FICA only ($7,664.92). That left out the NY PFL ($411.91) and DBL ($31.20) employee contributions, which the reference treats as mandatory employee state payroll taxes at their statutory maximums." -us,scenario_082,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Reported rounded FICA only ($7,665) and said NY imposes no counted employee payroll tax. The reference adds NY PFL at 0.432% capped at $411.91, plus DBL at $31.20." -us,scenario_082,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92 and treated NY employee payroll contributions as not modeled. The reference includes NY PFL ($411.91, at the 2026 cap) and DBL ($31.20) as mandatory employee state payroll taxes." -us,scenario_082,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Included both NY items but used the outdated 0.388% PFL rate on an ~$89,343 wage cap ($346.65). The 2026 PFL is 0.432% of wages with a $411.91 maximum, which binds at $100,195, so PFL came out $65.26 short." -us,scenario_082,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92, then added an arbitrary $129.39 'SDI/PFL estimate'. The correct NY add-on is PFL $411.91 (0.432% capped) plus DBL $31.20, or $443.11." -us,scenario_082,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92) on the view that no NY payroll tax applies. It left out NY PFL ($411.91) and DBL ($31.20), which the reference counts at the statutory maximum employee contribution." -us,scenario_082,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Subtracted the $181 traditional 401(k) deferral from FICA wages, giving $100,014. Elective deferrals are still Social Security and Medicare wages under IRC §3121(v)(1)(A), so the base is $100,195. It also left out NY PFL ($411.91) and DBL ($31.20)." -us,scenario_082,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Added NY DBL ($31.20) but left out the larger NY PFL employee contribution: 0.432% of $100,195, capped at the 2026 maximum of $411.91. This undercounted NY payroll tax by $411.91." -us,scenario_082,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,Used an outdated PFL maximum of $369.68 instead of the 2026 maximum of $411.91 (0.432% rate). It also left out the $31.20 DBL contribution. -us,scenario_082,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Applied the correct 0.432% PFL rate but skipped the 2026 annual maximum, reporting $432.84 instead of the capped $411.91. That overstated payroll tax by $20.93." -us,scenario_082,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Used about $333 for NY PFL, which is the older 2024 maximum, instead of the 2026 maximum of $411.91 (0.432% of wages, capped). It also rounded DBL to $31." -us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Reported only 7.65% FICA on $100,195 ($7,665). It left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts as mandatory state payroll taxes." -us,scenario_082,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premium input from FICA wages, giving $78,987. The listed premium is not a salary reduction of the $100,195 gross wages, so Social Security and Medicare apply to the full $100,195. It also figured PFL at an outdated 0.373% on the reduced base ($294.62) instead of the $411.91 cap." -us,scenario_082,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Cut FICA wages to $78,987 by subtracting the ESI premium input. Employee Social Security and Medicare apply to the full $100,195 gross wages. It also used about $394 for PFL instead of $411.91 and left out DBL ($31.20)." -us,scenario_082,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92). It left out NY PFL ($411.91) and DBL ($31.20), which the reference counts at the statutory maximum as mandatory employee state payroll taxes." -us,scenario_082,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Computed FICA on $78,987 after subtracting the $21,208 ESI premiums. The FICA base is the full $100,195 ($7,664.92 FICA), and NY PFL/DBL add $443.11." -us,scenario_082,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Treated $78,987 (gross wages minus ESI premiums) as FICA-taxable wages. Employee Social Security and Medicare apply to the full $100,195, and NY adds PFL $411.91 plus DBL $31.20." -us,scenario_082,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Computed Social Security plus Medicare only ($7,664.92) and left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts." -us,scenario_082,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Used the outdated 0.388% PFL rate on an assumed $96,200 wage cap ($373.26). The 2026 PFL rate is 0.432% with a $411.91 annual maximum, so PFL came out $38.65 short. DBL ($31.20) was correct." -us,scenario_082,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Listed components of $6,212 + $1,453 = $7,665 but reported $7,765, a $100 addition error. It also left out NY PFL ($411.91) and DBL ($31.20)." -us,scenario_082,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Said NY has no mandatory employee payroll tax and reported a rounded FICA-only $7,660, slightly below the exact $7,664.92. It left out NY PFL ($411.91) and DBL ($31.20), which the reference counts at their statutory maximums." -us,scenario_082,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Reported FICA only, rounded down to $7,659 (exact is $7,664.92). It left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference includes." -us,scenario_082,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Estimated combined NY PFL/DBL at $385.73. The 2026 PFL is 0.432% of wages capped at $411.91, plus DBL of $31.20, for $443.11, so it used an outdated PFL rate or cap." -us,scenario_082,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Included NY DBL ($31.20) but left out NY PFL, a 0.432% employee contribution that reaches the $411.91 annual maximum at $100,195 of wages." -us,scenario_082,payroll_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Reported 7.65% FICA only ($7,664.92). It left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference treats as mandatory state payroll taxes." -us,scenario_082,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Reported FICA only at $7,654, which is also $10.92 below the exact 7.65% × $100,195 = $7,664.92. It left out NY PFL ($411.91) and DBL ($31.20), which the reference includes." -us,scenario_082,payroll_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Applied the 0.432% PFL rate without the 2026 annual maximum, reporting $432.84 instead of the capped $411.91. That overstated the total by $20.93." -us,scenario_082,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Reported 7.65% FICA only ($7,665) and left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts." -us,scenario_082,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premiums from FICA wages ($78,987). Employee Social Security and Medicare apply to the full $100,195. It also explicitly said NY has no employee payroll tax, leaving out PFL ($411.91) and DBL ($31.20)." -us,scenario_082,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Used about $367 for NY PFL instead of the 2026 maximum of $411.91 (0.432% capped). It also said no other NY employee payroll tax applies, which left out the $31.20 DBL contribution." -us,scenario_082,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,665) and left out NY PFL ($411.91) and DBL ($31.20), which the reference includes at their statutory maximums." -us,scenario_082,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Subtracted both the ESI premiums ($21,208) and the 401(k) deferral ($181) from FICA wages, giving $78,806. Elective 401(k) deferrals are FICA wages, and the full $100,195 is the payroll-tax base. It also left out NY PFL/DBL ($443.11)." -us,scenario_082,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"Estimated NY PFL at about $365 instead of the 2026 maximum of $411.91 (0.432% of wages, capped). DBL ($31) was about right." -us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no payroll_tax value or explanation, so there was no answer to score." -us,scenario_082,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Computed FICA ($7,664.92), rounded it to $7,665, and left out the NY PFL ($411.91) and DBL ($31.20) employee contributions that the reference counts." -us,scenario_082,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,665) and never added the NY PFL ($411.91 at the 2026 cap) or DBL ($31.20) employee contributions that the reference includes." -us,scenario_082,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Tried several NY estimates and then settled on an unexplained negative ~$200 'NY adjustment', reporting $7,464.33, which is below FICA alone. NY PFL ($411.91) and DBL ($31.20) add $443.11 to the $7,664.92 FICA." -us,scenario_082,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Listed $6,212.09 + $1,452.83 + $0 Additional Medicare Tax but reported $7,752.42, an unexplained $87.50 extra. It also explicitly left out NY PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92 but called NY PFL/DBL negligible and left them out. Reading 1 counts them at the maximum employee amounts ($411.91 PFL + $31.20 DBL = $443.11). Reading 2 leaves them out because NY lets the employer collect less or none." +us,scenario_082,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92) and left out NY PFL ($411.91 at 0.432%, capped) and DBL ($31.20). Under the reading that counts maximum NY employee contributions as mandatory state payroll taxes, the total is $8,108.03. Under the employer-absorbs reading, it is $7,664.92." +us,scenario_082,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Used a 0.62% PFL rate on all wages ($621.21) and skipped the $411.91 annual cap; the 2026 rate is 0.432%. It also added a vague ~$19 for other items instead of the $31.20 DBL maximum, so it overstated the NY component by about $197." +us,scenario_082,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Reported rounded FICA only ($7,665) and left out NY PFL ($411.91) and DBL ($31.20). The prompt allows both readings: these NY contributions are mandatory employee payroll taxes, or they are optional for the employer to collect and so excluded." +us,scenario_082,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Included DBL at its $31.20 maximum but contradicted itself on PFL and settled on an unsupported $49.31. The reference counts the maximum the employer may collect: $411.91 of PFL (0.432% × $100,195 = $432.84, capped) plus $31.20 of DBL, or $443.11, against the model's $80.51; the employer may collect less or none." +us,scenario_082,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Said NY has no mandatory employee payroll tax and reported FICA only ($7,664.92), leaving out PFL ($411.91) and DBL ($31.20). This matches the reading where employer-optional collection makes those contributions non-mandatory, not the reading that counts the maximum employee contributions." +us,scenario_082,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Reported FICA only (about $7,665) and left out NY PFL ($411.91) and DBL ($31.20) as not mandatory. Under the reading that includes the maximum NY employee contributions, the total is $8,108.03." +us,scenario_082,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Computed FICA at $7,664.92 and assumed no NY employee payroll tax, which leaves out PFL ($411.91) and DBL ($31.20). Whether these count as ""mandatory employee state payroll taxes"" depends on the reading, since NY lets the employer absorb them." +us,scenario_082,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Included DBL at its $31.20 maximum but used the 2025 PFL rate of 0.388% with a cap of about $89,343, yielding $346.65. The reference counts the maximum collectible PFL share at the 2026 rate of 0.432%, capped at $411.91, so the model's PFL amount is $65.26 below the reference; the employer may collect less or none." +us,scenario_082,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Added only a guessed ~$129.39 for NY SDI/PFL. The reference counts the maximum the employer may collect: PFL at the $411.91 cap (0.432% × $100,195 = $432.84, capped) plus DBL at $31.20, for $443.11; the employer may collect less or none." +us,scenario_082,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92) and left out NY PFL ($411.91) and DBL ($31.20). That is the reading where NY employee contributions are optional, not the one that includes the maximum employee contributions." +us,scenario_082,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Subtracted the $181 traditional 401(k) deferral from FICA wages, but elective deferrals stay subject to Social Security and Medicare, so the base is the full $100,195. It also said NY has no employee payroll tax, leaving out PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Included the NY DBL employee contribution ($31.20) but dropped Paid Family Leave. The reference also includes PFL at the maximum the employer may collect, $411.91 (0.432% of wages, capped); the employer may collect less or none of either contribution." +us,scenario_082,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,Used an outdated PFL maximum of $369.68 instead of the 2026 cap of $411.91. It also left out the NY DBL contribution of $0.60/week × 52 = $31.20. +us,scenario_082,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Applied the correct 0.432% PFL rate to all $100,195 ($432.84) but missed the $411.91 annual maximum. That overstated PFL by $20.93." +us,scenario_082,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Used about $333 for NY PFL and rounded DBL to $31, giving about $364 of NY contributions. The reference counts the maximum the employer may collect in 2026, $411.91 of PFL (0.432% rate, capped) plus $31.20 of DBL, or $443.11; the employer may collect less or none." +us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA to $100,195 ($7,665) and left out NY PFL ($411.91) and DBL ($31.20). That fits the reading where these employer-optional contributions are not mandatory, not the reading that counts the maximum employee amounts." +us,scenario_082,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premium input from FICA wages and applied 7.65% to $78,987. The listed gross wages of $100,195 are the FICA base. It also used 0.373% for PFL on the reduced base ($294.62) instead of the 2026 rate of 0.432%, capped at $411.91." +us,scenario_082,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Took the $21,208 ESI premiums out of wages, leaving a FICA base of $78,987 instead of the full $100,195, which understated SS and Medicare by about $1,622. It also used ~$394 for PFL and no DBL, compared with the reference's maximum collectible shares of $411.91 and $31.20; the employer may collect less or none." +us,scenario_082,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Reported only federal SS and Medicare on gross wages ($7,664.92), with no NY PFL ($411.91) or DBL ($31.20). This matches the reading where those employer-optional contributions are excluded." +us,scenario_082,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Applied FICA to $78,987 after subtracting ESI premiums, but the FICA base is the full $100,195 of gross wages. It also put about $270 on NY SDI/PFL, below the reference's $443.11 maximum collectible share ($411.91 PFL cap + $31.20 DBL); the employer may collect less or none." +us,scenario_082,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Reduced FICA wages to $78,987 by subtracting the $21,208 ESI premiums, but SS and Medicare apply to the full $100,195. It also used about $334 for NY SDI/PFL, below the reference's $443.11 maximum collectible share; the employer may collect less or none." +us,scenario_082,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly ($7,664.92) but left out NY PFL ($411.91) and DBL ($31.20). That follows the reading where employer-optional NY contributions are not mandatory, not the one that includes the maximum employee amounts." +us,scenario_082,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Included DBL at its $31.20 maximum but computed PFL with the 2025 rate (0.388% × ~$96,200 = $373.26). The reference counts the maximum collectible PFL share at the 2026 rate of 0.432%, capped at $411.91, so the model's PFL amount is $38.65 below the reference; the employer may collect less or none." +us,scenario_082,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Its own figures were $6,212 + $1,453 = $7,665, but it reported $7,765, a $100 arithmetic slip. It also said NY has no employee payroll tax, leaving out PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Reported FICA only, rounded down to $7,660 from $7,664.92, and left out NY PFL ($411.91) and DBL ($31.20). This follows the reading where NY employee contributions are optional for the employer, so they are not mandatory taxes." +us,scenario_082,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Reported FICA only, rounded to $7,659 instead of $7,664.92, and left out NY PFL ($411.91) and DBL ($31.20). This matches the reading that excludes employer-optional NY contributions." +us,scenario_082,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Included NY PFL/DBL but estimated them together at $385.73. The reference counts the maximum the employer may collect in 2026: PFL at the $411.91 cap (0.432% rate) plus DBL at $31.20, for $443.11; the employer may collect less or none." +us,scenario_082,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Added the NY disability contribution ($31.20) but left out the NY Paid Family Leave contribution, which has the same status. The reference also counts the maximum collectible PFL share of $411.91: 0.432% of $100,195 is $432.84, capped at $411.91; the employer may collect less or none of either contribution." +us,scenario_082,payroll_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Applied 7.65% to $100,195 ($7,664.92) and left out NY PFL ($411.91) and DBL ($31.20). That is the reading where employer-optional NY contributions are not mandatory, not the one that counts the maximum employee amounts." +us,scenario_082,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Reported FICA only, slightly understated at $7,654 versus $7,664.92, and left out NY PFL ($411.91) and DBL ($31.20). This follows the reading that excludes employer-optional NY employee contributions." +us,scenario_082,payroll_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Applied the 0.432% PFL rate to all wages ($432.84) but missed the $411.91 annual maximum, overstating PFL by $20.93." +us,scenario_082,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA to wages ($7,665) and left out NY PFL ($411.91) and DBL ($31.20). That matches the reading where employer-optional NY contributions are not mandatory employee taxes." +us,scenario_082,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Treated the $21,208 ESI premium input as a pre-tax deduction from FICA wages, giving a $78,987 base instead of the full $100,195 of gross wages. It also said NY has no employee payroll tax, leaving out PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Used about $367 for PFL instead of the 2026 maximum of $411.91 (0.432% rate). It also said no other NY employee payroll tax applies, leaving out the $31.20 DBL contribution." +us,scenario_082,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly (about $7,665) and left out NY PFL ($411.91) and DBL ($31.20). That follows the reading that excludes employer-optional NY employee contributions." +us,scenario_082,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Subtracted both the $21,208 ESI premiums and the $181 traditional 401(k) deferral from FICA wages, giving $78,806. The base is the full $100,195; 401(k) deferrals are always FICA-taxable. It also left out NY PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"Estimated NY PFL at about $365, an outdated amount, instead of the 2026 cap of $411.91 (0.432% rate). It also rounded SDI to $31, giving about $396 of NY contributions against the reference's $443.11 maximum collectible share; the employer may collect less or none." +us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or reasoning came back, so the output is missing." +us,scenario_082,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Computed FICA at $7,664.92 (reported $7,665) and left out NY PFL ($411.91) and DBL ($31.20). That matches the reading where employer-optional NY contributions are not mandatory." +us,scenario_082,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Reduced FICA wages to $78,987 by subtracting ESI premiums, though FICA applies to the full $100,195, which understated SS and Medicare by about $1,622. It also used $372.40 for NY contributions, below the reference's $443.11 maximum collectible share; the employer may collect less or none." +us,scenario_082,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Reported FICA only (about $7,665) and left out NY PFL ($411.91) and DBL ($31.20). This follows the reading where employer-optional NY contributions are not counted as mandatory." +us,scenario_082,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"At one point added NY PFL and SDI to FICA, then subtracted an invented ~$200 ""NY deduction"" from $7,664.92 and reported $7,464.33. The reference adds the maximum the employer may collect, $443.11 ($411.91 PFL cap + $31.20 DBL); the employer may collect less or none." +us,scenario_082,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Its own components were $6,212.09 + $1,452.83 = $7,664.92, but it reported $7,752.42, an $87.50 arithmetic error. It also left out NY PFL ($411.91) and DBL ($31.20)." us,scenario_082,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It computed about $5,891 of pre-credit tax on $105,454 using rates higher than the 2026 HOH schedule, which gives $5,442.63 of main tax plus $156.51 of supplemental tax. It then subtracted about $396 for New York's child and dependent care credit. That credit is refundable in New York, so it doesn't reduce tax before refundable credits. The model also kept the $68 SALT refund in NY AGI instead of subtracting it." -us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used the correct 2026 HOH main schedule, but it computed the supplemental recapture by flattening taxable income at 5.4%, which gave only $49.75 instead of $156.51. Above $107,650 of NY AGI, the recapture uses the higher rate of the bracket that starts at $107,650. It also kept the $68 SALT refund in NY AGI and deducted the $8 IRA contribution, which is nondeductible with a workplace plan at this income." -us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It set NY taxable income equal to a mis-summed federal taxable income of $62,149 after the $14,600 federal standard deduction. The correct path starts from NY AGI of $117,593.62 and subtracts the $11,200 NY HOH standard deduction and the $1,000 dependent exemption, giving $105,393.62. Its $5,649 doesn't follow from any NY schedule applied to its own $62,149 base." -us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave a rounded $5,200 with no derivation. The 2026 HOH schedule on $105,393.62 yields $5,442.63, and NY AGI above $107,650 adds $156.51 of supplemental tax. Its figure leaves out the supplemental tax and understates the bracket tax as well." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It filed the head as single with the $8,000 standard deduction and no $1,000 dependent exemption, instead of head of household with the $11,200 deduction. It also added the $2,289 short-term gain on top of the $3,000 capped loss, even though that gain is already netted into the capital loss. After applying the pre-2026 single schedule, it took about $1,145 off its $6,221 figure without any credit that applies at this income." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used single status with an $8,000 standard deduction and no dependent exemption, instead of the HOH $11,200 deduction plus the $1,000 exemption. It also skipped the $181 traditional 401(k) exclusion. It then subtracted an unexplained ~$1,050 for household and child-care credits, although the household credit is phased out at this AGI and NY's child-care credit is refundable." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used an $8,000 standard deduction for head of household, but New York's HOH amount is $11,200, and it left out the $1,000 dependent exemption. Together these pushed taxable income up to about $109,654. It then applied a rough 5.5–6% marginal estimate instead of the 2026 HOH schedule (3.9%/4.4%/5.15%/5.4%) plus the $156.51 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its taxable income of $105,386 was essentially right, but its main tax came to $5,808.15 instead of $5,442.63. It cut only the lower brackets and didn't apply the 2026 rate of 5.4% across the $20,900–$107,650 HOH bracket. Its $81.40 supplemental recapture also falls short of the correct $156.51." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It treated the head as a single filer and itemized $9,917. Head-of-household status gives an $11,200 standard deduction, which exceeds the $9,328 of NY itemized deductions, plus a $1,000 dependent exemption. It also used the outdated 5.85% single schedule and subtracted $120 for New York's child care credit, which is refundable and doesn't reduce tax before refundable credits." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used single status and itemized $9,917 instead of taking the HOH $11,200 standard deduction plus the $1,000 dependent exemption. It then applied a 6.25% single bracket. Finally, it cut about $1,393 for a household credit that is fully phased out when NY AGI is about $117,600." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it estimated the tax at about $5,400 using a loose 4–5.5% range. The 2026 HOH schedule gives exactly $5,442.63. It also left out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its $86,949 federal AGI deducts essentially the entire $30,711 net capital loss, but the annual deduction is capped at $3,000. It then used a ~$10,000 standard deduction and $2,000 of exemptions, although New York allows only the $1,000 dependent exemption. It also subtracted the Empire State child credit, which is refundable." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It claimed $20,691 of NY itemized deductions, far more than the actual $9,328. Because $9,328 is less than the $11,200 HOH standard deduction, the standard deduction applies, so taxable income is $105,393.62 rather than its $95,971. It also used bracket widths that don't match 2026, applied 5.5% as the top rate, and left out the $156.51 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It took the $11,200 standard deduction but dropped the $1,000 dependent exemption. Its pre-credit figure of $5,731.77 doesn't match the 2026 schedule, which gives $5,442.63 of main tax plus $156.51 of supplemental tax. It then subtracted $120 for NY's child and dependent care credit as if it were nonrefundable, though New York's version is refundable." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $2,000 of exemptions, but New York has no personal exemption for the head, only $1,000 per dependent, so it understated taxable income by $1,000. It also left out the $156.51 supplemental recapture required when NY AGI exceeds $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a $10,500 standard deduction instead of the $11,200 HOH amount and left out the $1,000 dependent exemption. It then subtracted a $30 household credit, which isn't available when NY AGI is about $117,600. It also left out the $156.51 supplemental recapture." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It taxed only wage income after the standard deduction. That leaves out $20,480 of dividends and $100 of interest, which net of the $3,000 capital-loss deduction lift NY AGI to $117,593.62. It also missed the $156.51 supplemental tax that applies once NY AGI exceeds $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its taxable income of $84,178 is about $21,216 below the correct $105,393.62, which matches subtracting the $21,208 of employer-sponsored insurance premiums plus the $8 IRA from income. Neither reduces the listed gross wages in this computation. Taking the premiums out also dropped NY AGI below $107,650 and removed the $156.51 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 is about $21,140 below the correct $117,593.62, which matches subtracting the $21,208 of ESI premiums as a pre-tax wage exclusion that doesn't apply here. That pushed NY AGI below the $107,650 supplemental-tax threshold and dropped the $156.51 recapture. Its taxable income of $79,254 also doesn't match the stated $11,200 deduction and $1,000 exemption." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Its $6,040.50 is $441 above the correct figure: the 2026 HOH schedule on $105,393.62 gives $5,442.63, plus $156.51 of supplemental tax. The overstatement matches taxing NY AGI as a single filer with the $8,000 standard deduction and no dependent exemption, instead of HOH with the $11,200 deduction and the $1,000 exemption." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,220 is about $1,379 below the correct $5,599.14, which matches taking roughly $21,000 out of taxable income, the size of the $21,208 ESI premiums. Those premiums don't reduce income here. Removing them also dropped NY AGI below $107,650 and eliminated the $156.51 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,591 is $2,008 below the correct $5,599.14, which means about $37,000 of taxable income is missing from the correct $105,393.62. That gap matches treating items such as the $21,208 ESI premiums or the uncapped $30,711 capital loss as deductions, when only a $3,000 capital loss is allowed." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the correct $11,200 HOH deduction and $1,000 exemption. However, its $5,533 reflects pre-2026 rates: 5.5% on the $20,900–$107,650 bracket instead of 2026's 5.4%, with matching higher rates on the lower brackets. It also left out the $156.51 supplemental tax triggered by NY AGI above $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $5,599.14 reference." -us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It built $27,966 of NY itemized deductions from two errors. It included about $4,700 of NY income tax, which New York requires adding back when itemizing. It also claimed $13,349 of medical expenses, treating the ESI premiums as deductible, when only $1,965 of out-of-pocket medical exists and it falls below the 7.5% floor. Because itemized deductions are only $9,328, the $11,200 HOH standard deduction applies; the model also skipped the $1,000 dependent exemption and subtracted NY's refundable child-care credit." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave a rough estimate of $5,150 without applying the 2026 HOH schedule. That schedule yields $5,442.63 on $105,393.62 of taxable income, and NY AGI above $107,650 adds $156.51 of supplemental tax. Its figure leaves out the supplemental tax and understates the bracket tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no derivation, and its $7,777 is more than New York would charge even with zero deductions. The 2026 HOH schedule on all of the $117,593.62 NY AGI plus the recapture comes to well under $7,000. The correct path is the $11,200 standard deduction and $1,000 exemption, then the 2026 HOH schedule, giving $5,442.63 plus $156.51 of supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it applied the pre-2026 HOH rates of 4%, 4.5%, 5.25% and 5.5%, which give $5,548. The 2026 rates of 3.9%, 4.4%, 5.15% and 5.4% give $5,442.63. It also left out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,954.80 corresponds to roughly $55,000 of taxable income, about half the correct $105,393.62. That is the scale of deducting both the uncapped $30,711 capital loss and the $21,208 ESI premiums, though only a $3,000 loss is allowed and the premiums don't reduce income. It also dropped the $156.51 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of roughly $106,394 leaves out the $1,000 dependent exemption, which brings the correct figure to $105,393.62. Its $5,815 also exceeds the 2026 HOH result, $5,442.63 of main tax plus $156.51 of supplemental tax, because it applied rates higher than 2026's 5.4% in the $20,900–$107,650 bracket." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It took the $11,200 standard deduction but left out the $1,000 dependent exemption. Its $5,977.08 also overstates the 2026 HOH tax. Applying 3.9%, 4.4%, 5.15% and 5.4% to $105,393.62 gives $5,442.63, and the supplemental recapture adds only $156.51." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its taxable income of $92,961.65 implies about $23,632 of NY itemized deductions, which is consistent with treating the $21,208 ESI premiums as deductible medical expenses. Actual NY itemized deductions are $9,328.22, so the $11,200 HOH standard deduction applies, giving $105,393.62 of taxable income." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Its $5,458 is essentially the 2026 HOH main-schedule tax alone ($5,442.63 on $105,393.62). It left out the $156.51 supplemental recapture that applies because NY AGI of $117,593.62 exceeds $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted $120 for the NY child and dependent care credit as if it were nonrefundable, but New York's credit is refundable and doesn't reduce tax before refundable credits. Its pre-credit figure of $5,665.24 also exceeds the correct $5,599.14 ($5,442.63 under the 2026 HOH schedule plus $156.51 of supplemental tax)." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $92,961.65 implies about $23,632 of NY itemized deductions, which is consistent with treating the $21,208 ESI premiums as deductible medical expenses. Actual NY itemized deductions are $9,328.22, so the $11,200 HOH standard deduction applies and taxable income is $105,393.62." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave a rounded $5,500 without applying the actual schedule. The 2026 HOH rates (3.9%, 4.4%, 5.15%, 5.4%) on $105,393.62 give $5,442.63, and NY AGI above $107,650 adds $156.51 of supplemental tax, which its estimate leaves out." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 subtracts the $21,208 ESI premiums, which don't reduce income here, so the correct NY AGI is $117,593.62. It used a $12,100 HOH standard deduction instead of $11,200 and subtracted NY's refundable child-care credit as if it were nonrefundable. With AGI understated, it also missed the $156.51 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it applied the pre-2026 formula of $901 plus 5.5% over $20,900. The 2026 rates of 3.9%, 4.4%, 5.15% and 5.4% give $5,442.63. It also left out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It itemized deductions even though the $11,200 HOH standard deduction exceeds the $9,328 of NY itemized deductions. It also taxed the result on the single schedule ($4,271 plus 6% over $80,650) instead of the 2026 HOH schedule. It left out the $156.51 supplemental tax as well." -us,scenario_082,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It left out the $1,000 dependent exemption, giving $106,386 of taxable income instead of $105,393.62. It then used pre-2026 rates with the wrong HOH bracket widths (5,400 and 3,000 instead of 4,850 and 3,250). It also skipped the $156.51 supplemental tax. Its $5,598 comes from offsetting errors rather than from the $5,442.63 main tax plus $156.51 supplemental." -us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It subtracted $2,000 of exemptions, but New York has no personal exemption for the head, only $1,000 per dependent. It also didn't subtract the $68 SALT refund from NY AGI. It then took off $120 for NY's child care credit, which is refundable and doesn't reduce tax before refundable credits." -us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $5,599.14 reference." -us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its taxable income of $105,394 was correct, but it used a rate of about 5.7% for the bracket containing that income. Under the 2026 HOH schedule, the $20,900–$107,650 bracket is taxed at 5.4%, which gives $5,442.63. It also didn't compute the separate $156.51 supplemental recapture." -us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the $33,000 long-term capital loss as wiping out most of the income. The net capital loss is limited to a $3,000 deduction, so NY AGI is still $117,593.62. It also used a single $8,000 deduction and pointed to a CTC, which doesn't reduce NY tax before refundable credits." -us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used $8,000 as the HOH standard deduction, but New York's HOH amount is $11,200. It also applied pre-2026 rates, including 6.25% above $107,650. Finally, it subtracted $120 for NY's child-care credit, which is refundable and doesn't reduce tax before refundable credits." -us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $30,711 net capital loss, giving federal AGI of $89,951, when the annual deduction is capped at $3,000 and federal AGI is $117,661.12. It used single status with the $8,000 deduction instead of HOH with $11,200. Its final $4,755.44 is higher than its own $4,292.89 pre-credit figure, so the closing step is arithmetically inconsistent." -us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $139,586 ignores the $3,000 capital-loss deduction. It then subtracted an invented 'federal deduction adjustment' and used single status with the $8,000 deduction instead of HOH with $11,200. It also subtracted the refundable Empire State Child Credit, a household credit that is phased out at this income, and a NY CTC that doesn't exist." -us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the pre-2025 ESCC formula (33% of a $1,000 pre-TCJA base = $330) instead of the enhanced $1,000 under-4 credit, which a $16.50-per-$1,000 phase-out above $75,000 reduces to $307. It also guessed about $150 for the care credit instead of applying the 2026 §606(c-2) percentage of about 29.35% to $3,000 of expenses, which gives $880.55." -us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Computed the care credit as 20% of the $600 federal CDCC ($120). From 2026, §606(c-2) instead applies an AGI-based percentage of about 29.35% directly to the $3,000 expense cap, giving $880.55. It also applied the ESCC phase-out as a continuous 1.65% of the excess ($703.79) rather than $16.50 per full $1,000 (42 × $16.50 = $693), so it got $296.21 instead of $307." -us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly said NY has no refundable child credit. The refundable Empire State Child Credit is $307 after phase-out, and the refundable 2026 §606(c-2) child care credit is $880.55, so refundable credits total $1,187.55, not $0." -us,scenario_082,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Said no refundable NY credits apply at this income, with no derivation. The $1,000 under-4 ESCC is only partly phased out ($307 remains), and the 2026 §606(c-2) child care credit is refundable at $880.55." -us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Made up a $110 under-4 ESCC amount instead of the enhanced $1,000 credit less the $16.50-per-$1,000 phase-out above $75,000 ($307). It also wrongly treated the NY child care credit as nonrefundable above about $50K, dropping the $880.55 refundable §606(c-2) credit." -us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Assumed the ESCC and care credit fully phase out or become nonrefundable at this income. In fact the $1,000 under-4 ESCC keeps $307 after a $693 reduction, and the 2026 §606(c-2) care credit refunds $880.55 (about 29.35% of $3,000)." -us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Concluded income was too high for any NY refundable credit. At federal AGI of about $117.7K the under-4 ESCC is still $307, and the 2026 §606(c-2) child care credit is still $880.55." -us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Used the pre-2026 care credit of 20% of the federal $600 ($120). §606(c-2) instead applies about 29.35% to $3,000 of expenses, giving $880.55. It also rounded the ESCC phase-out up to 43 increments ($709.50) instead of counting 42 full $1,000 increments ($693), so it got $290.50 instead of $307." -us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Computed the ESCC as 33% of a $2,000 federal CTC ($660) with no phase-out. The 2026 under-4 credit is a flat $1,000 cut by $16.50 per $1,000 of AGI over $75,000, giving $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) percentage on $3,000 ($880.55), and understated AGI by subtracting the full $33,000 long-term loss rather than the $3,000 cap." -us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Assumed the ESCC is fully phased out and left out the care credit entirely. The $1,000 under-4 ESCC keeps $307, and the 2026 §606(c-2) refundable child care credit adds $880.55." -us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed the ESCC is fully phased out, but the $1,000 under-4 credit keeps $307 after 42 × $16.50 = $693 of reduction. It also used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) percentage of about 29.35% on $3,000 ($880.55)." -us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Only checked the NY EITC and missed the two refundable credits that apply: the Empire State Child Credit ($1,000 less a $693 phase-out = $307) and the 2026 §606(c-2) child care credit ($880.55)." -us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Left out the $307 Empire State Child Credit for the under-4 child. It also computed the care credit as 20% of the federal $600 ($120) instead of applying the 2026 §606(c-2) percentage of about 29.35% to $3,000 ($880.55)." -us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Applied the old rule that excludes children under 4 from the ESCC. Since 2025 a child under 4 gets a $1,000 credit, which phases down to $307 here. It also left out the refundable 2026 §606(c-2) child care credit of $880.55." -us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Gave the full $1,000 ESCC without the $16.50-per-$1,000 phase-out above $75,000 of federal AGI, which removes $693 and leaves $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) percentage on $3,000 ($880.55)." -us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the old $330 ESCC instead of the phased $1,000 under-4 credit ($307) and guessed $300 for the care credit instead of the §606(c-2) amount of $880.55. It also added an NYC School Tax Credit, which does not apply because the household is not stated to be in NYC." -us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Counted only a $330 ESCC from the pre-2025 33% formula. It missed the enhanced $1,000 under-4 credit phased to $307, and left out the $880.55 refundable 2026 child care credit." -us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Used the $100 ESCC minimum instead of the enhanced $1,000 under-4 credit less a $693 phase-out ($307). It also used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) percentage on $3,000 ($880.55)." -us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Applied the 110% low-income multiplier from the old federal-linked care credit table ($660), which does not fit this AGI or the 2026 §606(c-2) structure (about 29.35% of $3,000 = $880.55). It also used the old $330 ESCC instead of the phased $1,000 under-4 credit ($307)." -us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported zero refundable credits with no derivation, missing the $307 under-4 Empire State Child Credit and the $880.55 refundable 2026 §606(c-2) child care credit." -us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Reported only the old $330 ESCC rather than the $1,000 under-4 credit reduced by $693 to $307, and left out the refundable 2026 child care credit of $880.55." -us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Counted only a $330 ESCC from the pre-2025 formula, instead of the $307 left from the $1,000 under-4 credit after phase-out. It also left out the $880.55 §606(c-2) child care credit." -us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Used the old 20%-of-federal care credit ($120) instead of the 2026 §606(c-2) percentage of about 29.35% on $3,000 ($880.55). It also assumed the ESCC phases out completely, though $307 of the $1,000 under-4 credit remains." -us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so there was no substantive answer to score against the $1,187.55 reference." -us,scenario_082,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Treated the enhanced ESCC as 100% of the federal CTC ($2,200). The under-4 credit is a flat $1,000 cut by $16.50 per $1,000 of AGI over $75,000, giving $307. It also left out the $880.55 refundable §606(c-2) child care credit." -us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable NY credits, missing that the under-4 child gets a $307 ESCC after phase-out and that the $9,600 of child care expenses produce a refundable 2026 §606(c-2) credit of $880.55." -us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Looked only at the NY EITC and assumed nothing else applied. The ESCC ($307 after phase-out) and the refundable 2026 child care credit ($880.55 on $3,000 of expenses) both apply from the listed child and childcare facts." -us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Computed the care credit as 20% of the $600 federal CDCC ($120) instead of the §606(c-2) percentage of about 29.35% on $3,000 ($880.55). It also wrongly said no ESCC remains, though $307 of the $1,000 under-4 credit survives." -us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Gave the full $1,000 under-4 ESCC without the $16.50-per-$1,000 phase-out above $75,000, which reduces it to $307. It also left out the $880.55 refundable 2026 child care credit." -us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Used the pre-2026 care credit of 20% of the federal $600 ($120) instead of the §606(c-2) amount of $880.55. It also rounded the ESCC phase-out up to 43 increments ($290.50) instead of 42 full $1,000 increments ($307)." -us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the old rule that excludes children under 4 from the ESCC. The enhanced credit gives a 1-year-old $1,000, phased to $307 here. It also left out the refundable 2026 §606(c-2) child care credit of $880.55." -us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) percentage of about 29.35% on $3,000 ($880.55). It also rounded the ESCC phase-out to 43 increments ($290.50) instead of 42 ($307)." -us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Wrongly said the ESCC is fully phased out, though $307 of the $1,000 under-4 credit remains. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) care credit of $880.55." -us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,Wrongly treated the NY dependent care credit as nonrefundable and the child credit as unavailable. The 2026 §606(c-2) care credit refunds $880.55 and the under-4 ESCC refunds $307. -us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Got close on the ESCC ($312 vs. $307) but used the old 20%-of-federal care credit ($120). §606(c-2) applies about 29.35% to the $3,000 expense cap, giving $880.55." -us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Found no refundable NY credits, missing the $307 under-4 Empire State Child Credit and the $880.55 refundable 2026 child care credit." -us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Wrongly treated the federal CTC as fully phased out at about $117.7K AGI (the head-of-household phase-out starts at $200,000). It then dropped the ESCC, which is $1,000 less $693 = $307, and left out the $880.55 refundable §606(c-2) care credit." -us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used the $100 ESCC minimum on the false premise that the federal CTC is phased out. The enhanced under-4 ESCC is $1,000 less $693 = $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) amount of $880.55." -us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Used a $660 ESCC (33% of a $2,000 CTC) instead of the $1,000 under-4 credit cut by $16.50 per $1,000 above $75,000 ($307). It also used 20% of the federal CDCC ($120) instead of the 2026 §606(c-2) amount of $880.55." -us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Only checked the EITC and missed both refundable NY credits that apply: the $307 under-4 ESCC and the $880.55 2026 child care credit. -us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the NY child care credit as fully absorbed by tax liability, but the 2026 §606(c-2) credit is refundable at $880.55. It also left out the refundable $307 ESCC for the under-4 child." -us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so there was no substantive answer to score against the $1,187.55 reference." -us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Used the old 2017-law-CTC ESCC path with its $100 minimum, missing the enhanced flat $1,000 under-4 credit, which phases down to $307. It also used 20% of the federal CDCC ($120) instead of the §606(c-2) care credit of $880.55." -us,scenario_082,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Assumed income exceeds the ESCC range, but the $1,000 under-4 credit keeps $307 at about $117.7K AGI. It also left out the $880.55 refundable 2026 child care credit." -us,scenario_082,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Applied a 5% ESCC phase-out rate. The actual reduction is $16.50 per full $1,000 above $75,000 (42 × $16.50 = $693), leaving $307. It also left out the $880.55 refundable §606(c-2) child care credit." -us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Looked only at the NY EITC and missed the $307 under-4 Empire State Child Credit and the $880.55 refundable 2026 child care credit. -us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Got close on the ESCC ($313 vs. $307) but left out the refundable 2026 §606(c-2) child care credit. That credit applies about 29.35% to $3,000 of expenses and adds $880.55." +us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model subtracted about $396 of NY child and dependent care credit as if it were nonrefundable, but that credit is refundable in New York and does not reduce tax before refundable credits. It also skipped the $68 SALT-refund subtraction and overstated the pre-credit tax at about $5,891, when the schedule tax plus recapture is $5,599.14." +us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The schedule tax of $5,445.89 is about right, but the supplemental recapture is wrong. The model used the 5.4% rate of the taxable-income bracket as the flat recapture rate. For NY AGI above $107,650, the rate is the higher-bracket rate, so the model's recapture of $49.75 should have been $156.51. It also failed to subtract the $68 state tax refund from NY AGI." +us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model set NY taxable income equal to a federal taxable income of $62,149, built with the $14,600 federal standard deduction and a mis-summed income total. It never applied the NY HOH $11,200 standard deduction or the $1,000 dependent exemption to NY AGI of $117,594. The $5,649 figure does not follow from any bracket calculation on its own stated base." +us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model gave no computation. The correct result is NY taxable income of $105,394 taxed at $5,442.63 under the 2026 HOH schedule, plus a $156.51 supplemental recapture because NY AGI exceeds $107,650. The $5,200 answer is consistent with a rough bracket estimate that leaves out the recapture tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model filed as single with an $8,000 standard deduction and no dependent exemption, when a parent with a qualifying child files head of household ($11,200 deduction plus a $1,000 exemption). It also added the $2,289 short-term gain separately instead of netting it against the long-term loss before the $3,000 cap. It then cut its own $6,221 figure to $5,076 without any stated rule." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used single filing status with the $8,000 standard deduction and no $1,000 dependent exemption instead of HOH ($11,200). It did not subtract the traditional 401(k) contribution from wages and applied a 6% rate to income above $80,650. It then reduced $6,326 to $5,276 through an unexplained adjustment, even though no nonrefundable credit applies." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an $8,000 standard deduction for head of household instead of the NY HOH amount of $11,200 and left out the $1,000 dependent exemption, which inflated taxable income to $109,654. It then applied rough 5.5–6% marginal rates instead of the 2026 HOH schedule plus the recapture." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The taxable income of $105,386 is close, but the model's 2026 HOH schedule tax of $5,808.15 is too high. The 3.9/4.4/5.15/5.4% brackets give about $5,442 on that base. Because the schedule tax was inflated, the 5.9%-flat-rate recapture came out at only $81.40 instead of $156.51." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model filed as single, compared itemized deductions of $9,917 against the $8,000 single standard deduction instead of the larger HOH $11,200, and left out the $1,000 dependent exemption. It used an outdated single schedule with a 5.85% rate. It then subtracted a $120 NY child care credit, which is refundable in New York and does not reduce tax before refundable credits." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used single filing status, itemized $9,917 against an $8,000 single standard deduction instead of taking the HOH $11,200, and left out the dependent exemption. It then cut its $6,193 tax to $4,800 by citing a household credit that is fully phased out at this income." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model reached the correct taxable income of $105,394, then applied rough 4–5.5% rates and left out the $156.51 supplemental recapture tax that applies because NY AGI exceeds $107,650. The correct total is $5,442.63 + $156.51 = $5,599.14." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model deducted the full $30,711 net capital loss, giving AGI of about $86,949, instead of capping the loss at $3,000. It also claimed $2,000 of exemptions when NY allows none for the filer. It then subtracted the Empire State Child Credit, which is refundable and does not reduce tax before refundable credits." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model claimed NY itemized deductions of $20,691, but allowable NY itemized deductions are only $9,328, so the $11,200 HOH standard deduction applies. That understated taxable income at $95,971 instead of $105,394. It also used pre-2026 rates and no recapture tax." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model subtracted a $120 NY child and dependent care credit, which is refundable in New York and does not reduce tax before refundable credits. It also left out the $1,000 dependent exemption, taxing $106,394 instead of $105,394." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model claimed $2,000 of exemptions, but NY allows $1,000 only for the dependent child and nothing for the filer. It also deducted the nondeductible $8 IRA contribution. Its $5,408.73 is schedule tax only, leaving out the $156.51 supplemental recapture that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a $10,500 standard deduction instead of the $11,200 NY HOH amount and left out the $1,000 dependent exemption. It also omitted the supplemental recapture tax. It then subtracted a $30 household credit that is fully phased out at this income." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's own explanation taxes only wage income after the standard deduction, leaving out $20,480 of dividends and $100 of interest. The correct base is NY AGI of $117,594, which leaves $105,394 of taxable income after the $11,200 HOH deduction and $1,000 exemption. The $4,380 answer matches that understated income base and leaves out the recapture." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Taxable income of $84,178 matches subtracting the $21,208 employer-sponsored insurance premiums from wages. Those premiums are not a reduction of the listed gross wages. The correct NY taxable income is $105,394, and NY AGI above $107,650 triggers the $156.51 recapture." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's NY AGI of $96,454 subtracts the $21,208 employer-sponsored insurance premiums from wages, which is wrong because NY AGI is $117,594. It then made a further $5,000 arithmetic error, since $96,454 − $11,200 − $1,000 is $84,254, not $79,254." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no computation. The correct result is NY taxable income of $105,394 taxed at $5,442.63 under the 2026 HOH schedule, plus a $156.51 recapture, for $5,599.14. The $6,040.50 answer is about $441 too high, which is consistent with a single-status or smaller standard deduction, a missing dependent exemption, or pre-2026 higher rates." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model gave no figures. The $4,220 answer matches the tax on roughly $84,000 of taxable income, which is the result of wrongly subtracting the $21,208 ESI premiums from wages. The correct NY taxable income is $105,394, with a $156.51 recapture on top of the $5,442.63 schedule tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model gave no computation. Its $3,591 is about $2,000 below the correct $5,599.14, which matches taxing income far below the correct $105,394. That happens if the ESI premiums or the uncapped $30,711 capital loss are deducted, or if refundable credits are netted against tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied the right $11,200 HOH deduction and $1,000 exemption. Its $5,533 matches the pre-2026 4/4.5/5.25/5.5% schedule with no supplemental recapture, instead of $5,442.63 at the 2026 reduced rates plus $156.51 of recapture for NY AGI above $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value for state_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model's itemized deductions included about $4,700 of NY income tax, which NY does not allow as a deduction on its own return. They also included $13,349 of 'medical' that folds in the $21,208 ESI premiums, which are not a deductible medical expense here. Actual NY itemized deductions are $9,328, below the $11,200 HOH standard deduction. The model also subtracted the refundable $120 NY child care credit." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model's $5,150 rests only on 'standard deduction and limited credits'. It leaves out the $156.51 supplemental recapture for NY AGI above $107,650 and nets credits that are either refundable or phased out. The correct total is $5,442.63 of schedule tax on $105,394 plus $156.51." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model's $7,777 is a placeholder-style figure with no bracket calculation. It is about $2,178 above the correct $5,599.14, which comes from $105,394 of HOH taxable income at 2026 rates plus the $156.51 recapture." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model got the correct taxable income of $105,394 but applied the pre-2026 HOH rates (4/4.5/5.25/5.5%), giving $5,548. The correct 2026 rates of 3.9/4.4/5.15/5.4% give $5,442.63. The model also left out the $156.51 supplemental recapture tax that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model gave no computation. Its $2,954.80 matches roughly half the correct taxable income. The correct derivation taxes $105,394, which is NY AGI of $117,594 less the $11,200 HOH deduction and $1,000 exemption, at 2026 HOH rates for $5,442.63, then adds the $156.51 recapture." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Taxable income of $106,394 leaves out the $1,000 NY dependent exemption for the child. The tax applied to that base is also higher than the 2026 HOH schedule plus recapture would produce." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model applied only the $11,200 standard deduction and left out the $1,000 dependent exemption. Its $5,977.08 is about $378 above the correct figure, which matches applying pre-2026 or higher marginal rates instead of the 2026 3.9/4.4/5.15/5.4% schedule plus the $156.51 recapture." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model itemized about $23,632 of deductions, giving taxable income of $92,961.65. Allowable NY itemized deductions are only $9,328, so the $11,200 HOH standard deduction applies and taxable income is $105,394. The inflated itemized figure counts items such as the ESI premiums or gross medical costs that are not deductible." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied the HOH deduction, the exemption and the 2026 brackets, landing at about the $5,442.63 schedule tax. It left out the $156.51 supplemental recapture tax that applies because NY AGI of $117,594 exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model subtracted a $120 NY child and dependent care credit as nonrefundable, but New York makes that credit refundable, so it does not reduce tax before refundable credits. Its pre-credit tax of $5,665.24 also overstates the correct $5,442.63 schedule tax plus $156.51 recapture." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model itemized about $23,632 of deductions, giving taxable income of $92,961.65. Allowable NY itemized deductions are $9,328, below the $11,200 HOH standard deduction, so taxable income is $105,394. The supplemental tax built on that understated base also comes out too low." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no computation. Its $5,500 is close to the schedule tax on $105,394 but leaves out the $156.51 supplemental recapture for NY AGI above $107,650. The correct total is $5,442.63 + $156.51 = $5,599.14." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $96,454 by subtracting the $21,208 ESI premiums from wages and used a nonexistent $12,100 HOH standard deduction instead of $11,200. It then subtracted the $120 NY child care credit, which is refundable and does not reduce tax before refundable credits." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model got the correct taxable income of $105,394 but used the pre-2026 formula of $901 plus 5.5%. Under the 2026 rate cuts the correct schedule tax is $5,442.63. The model also left out the $156.51 supplemental recapture triggered by NY AGI above $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model itemized about $23,000 of deductions, reaching taxable income of $93,545. Allowable NY itemized deductions are only $9,328, so the $11,200 HOH standard deduction applies. It also taxed the result on the single schedule ($4,271 plus 6% over $80,650) instead of the HOH schedule." +us,scenario_082,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model left out the $1,000 dependent exemption, giving taxable income of $106,386 instead of $105,394. It used pre-2026 rates with a wrong 4.5% bracket width and left out the $156.51 supplemental recapture. These errors largely cancel, leaving $5,598 instead of $5,599.14." +us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model claimed $2,000 of exemptions, but NY gives $1,000 only for the dependent and none for the filer. It then subtracted a $120 NY child care credit, which is refundable in New York and does not reduce tax before refundable credits." +us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for state_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model got the correct taxable income of $105,394 but applied a nonexistent 5.75% bracket rate. The 2026 HOH rate in that range is 5.4%, which gives $5,442.63. The model also left out the $156.51 supplemental recapture for NY AGI above $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model let the $33,000 long-term loss wipe out most income, but the net capital loss deduction is capped at $3,000. It also netted the federal Child Tax Credit against NY tax, which is not allowed. Correctly, NY taxable income is $105,394 and the tax is $5,599.14, not zero." +us,scenario_082,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Taxable income of about $85,200 understates the correct $105,394 by roughly the $21,208 ESI premiums. Those premiums are not a reduction of the listed gross wages. The model also subtracted the refundable $120 NY child care credit." +us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used an $8,000 standard deduction for head of household instead of $11,200 and applied an outdated schedule with 5.85% and 6.25% rates. It then subtracted the $120 NY child care credit, which is refundable and does not reduce tax before refundable credits." +us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the full $30,711 net capital loss, giving AGI of $89,951, instead of capping it at $3,000. It used the single $8,000 standard deduction instead of HOH $11,200. It then raised its own $4,292.89 to $4,755.44 by citing a 'household credit', which is incoherent because the household credit is phased out at this income and a credit only lowers tax." +us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Federal AGI of $139,586 ignores the $3,000 capped capital loss and overstates income. The model used single status with an $8,000 deduction and an invented 'federal deduction adjustment'. It then subtracted the Empire State Child Credit, which is refundable, and a household credit that is phased out at this income." +us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Computed the ESCC as 33% of a pre-TCJA $1,000 CTC ($330). It should have used the enhanced $1,000 under-4 amount less the 1.65% phase-out above $75,000, which gives $307. It also guessed about $150 for the care credit, when 606(c-2) applies about 29.35% to $3,000 of expenses for $880.55." +us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Got the ESCC structure nearly right ($1,000 less 1.65% of the excess over $75,000). However, it reduced the credit on the exact excess (yielding $296.21) instead of per full $1,000 step (42 × $16.50 = $693, leaving $307). The larger miss is the care credit: it used the old 20%-of-federal-CDCC rule ($120) instead of the 2026 606(c-2) formula, which gives about 29.35% × $3,000 = $880.55." +us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly stated that New York has no refundable child credit, so it missed the refundable Empire State Child Credit ($1,000 under-4 base less a $693 phase-out = $307). It also missed the refundable NY child and dependent care credit ($880.55 under 606(c-2))." +us,scenario_082,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Assumed income ruled out every NY refundable credit. At about $117.7k AGI, the Empire State Child Credit for a child under 4 is still $307 after the 1.65% phase-out, and the 606(c-2) care credit pays $880.55 on $3,000 of expenses." +us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Made up a $110 under-4 ESCC amount instead of the $1,000 base reduced by $16.50 per $1,000 above $75,000 ($307). It also wrongly treated the NY care credit as nonrefundable above about $50k, when the 606(c-2) credit is refundable and worth $880.55 here." +us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Claimed the ESCC phases out completely and the care credit is nonrefundable at this AGI. The $1,000 under-4 ESCC only loses $693 at $117.7k AGI, leaving $307, and the refundable 606(c-2) care credit is $880.55." +us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Asserted that income was too high for any NY refundable credit. The $1,000 under-4 ESCC is only partly phased out ($307 remains), and the 2026 care credit pays about 29.35% of $3,000 = $880.55." +us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Rounded the ESCC phase-out up to 43 thousands ($290.50) instead of counting full $1,000 steps (42 × $16.50 = $693, leaving $307). It also used the old 20%-of-federal-CDCC rule ($120) instead of the 606(c-2) formula, which applies about 29.35% to $3,000 of expenses for $880.55." +us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Used the old 33%-of-federal-CTC ESCC formula ($660) and explicitly denied any income phase-out. The current under-4 credit is $1,000 less $16.50 per $1,000 above $75,000, which gives $307. It also computed the care credit as 20% of the $600 federal credit ($120) instead of $880.55 under 606(c-2)." +us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Concluded that NY AGI around $92k was too high for the ESCC. The $1,000 under-4 ESCC phases out at only $16.50 per $1,000 above $75,000, leaving $307 at the actual $117.7k AGI. It also ignored the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed the ESCC was fully phased out, but the $1,000 under-4 credit keeps $307 after the $693 reduction. It also applied the pre-2026 20%-of-federal-CDCC rule ($120) instead of the 606(c-2) percentage of $3,000 in expenses ($880.55)." +us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Only checked the NY EITC and ignored the two refundable credits that apply: the Empire State Child Credit ($307 after phase-out of the $1,000 under-4 base) and the 606(c-2) child and dependent care credit ($880.55)." +us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Left out the Empire State Child Credit entirely ($307 after the phase-out). It also computed the care credit under the old 20%-of-federal-CDCC rule ($120) instead of the 2026 formula, about 29.35% × $3,000 = $880.55." +us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Applied an outdated age-4 minimum to the ESCC. Children under 4 now get a $1,000 base credit, which is $307 after phase-out here. It also left out the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Allowed the full $1,000 under-4 ESCC without the $16.50-per-$1,000 phase-out above $75,000, which removes $693 at $117.7k AGI. It also used the old 20%-of-federal-CDCC rule ($120) instead of the 606(c-2) credit of $880.55." +us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the outdated $330 ESCC instead of the $1,000 under-4 base less the $693 phase-out ($307). It guessed $300 for the care credit instead of about 29.35% × $3,000 = $880.55. It also added a NYC school tax credit, which does not apply because no NYC residence is given." +us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Reported only an old-formula $330 ESCC instead of $307 (the $1,000 under-4 base less a $693 phase-out). It also left out the refundable 606(c-2) child and dependent care credit of $880.55." +us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Used the $100 ESCC floor instead of the enhanced $1,000 under-4 credit less the $693 phase-out ($307). It also computed the care credit as 20% of the federal $600 ($120) instead of $880.55 under 606(c-2)." +us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Applied the 110%-of-federal care-credit rate, which belongs to the lowest income band, to a $117k AGI household and got $660. The 606(c-2) percentage at this AGI is about 29.35% of $3,000, or $880.55. It also used a $330 ESCC instead of the phased-down $1,000 under-4 amount ($307)." +us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned zero, which misses both refundable NY credits for this household: the Empire State Child Credit ($1,000 under-4 base less $693 = $307) and the 606(c-2) care credit ($880.55)." +us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Reported only a $330 ESCC based on the old 33%-of-CTC formula instead of $307 (the $1,000 under-4 base less a $693 phase-out). It also omitted the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Counted only a $330 old-formula ESCC and left out the refundable child and dependent care credit. Under 606(c-2), that credit applies about 29.35% to $3,000 of expenses for $880.55, and the ESCC is $307 after phase-out." +us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Used the pre-2026 rule (20% of the $600 federal CDCC = $120) instead of the 606(c-2) formula, which gives $880.55. It also wrongly treated the ESCC as phased out, when $307 of the $1,000 under-4 credit remains." +us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so there was nothing to score against the $1,187.55 reference." +us,scenario_082,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Treated the enhanced ESCC as 100% of a $2,200 federal CTC. The actual under-4 amount is a flat $1,000 reduced by $16.50 per $1,000 of AGI above $75,000, which gives $307. It also omitted the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable NY credits and missed both the Empire State Child Credit for the 1-year-old ($307 after phase-out) and the refundable 606(c-2) child care credit on $3,000 of expenses ($880.55)." +us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Considered only NY EITC eligibility, so it missed the ESCC ($1,000 under-4 base less $693 = $307) and the refundable 606(c-2) care credit ($880.55). Neither credit depends on an EITC-style earned-income test." +us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Applied the old 20%-of-federal-CDCC rule ($120) instead of 606(c-2), which applies about 29.35% directly to $3,000 of expenses for $880.55. It also wrongly zeroed the ESCC, when $307 of the $1,000 under-4 credit survives the phase-out." +us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Allowed the full $1,000 under-4 ESCC without the $16.50-per-$1,000 phase-out above $75,000 ($693 at this AGI, leaving $307). It also left out the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Rounded the ESCC phase-out up to 43 thousands ($290.50) instead of 42 full $1,000 steps (which leaves $307). It also used the pre-2026 20%-of-federal care-credit rule ($120) instead of the 606(c-2) credit of $880.55." +us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Wrongly treated a 1-year-old as too young for the ESCC. Children under 4 now get a $1,000 base credit, which is $307 after phase-out here. It also missed the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Overcounted the ESCC phase-out by rounding the excess up to 43 thousands ($290.50 instead of $307). It also used the old 20%-of-federal-CDCC rule ($120) instead of the 2026 606(c-2) percentage applied to $3,000 of expenses ($880.55)." +us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Wrongly zeroed the ESCC, when $307 remains after the $693 phase-out of the $1,000 under-4 credit. It also used the pre-2026 20%-of-federal care-credit rule ($120) instead of $880.55 under 606(c-2)." +us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Treated the NY dependent care credit as nonrefundable, but the 606(c-2) credit is refundable ($880.55). It also wrongly denied the ESCC, which is $307 after phase-out of the $1,000 under-4 base." +us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Estimated the ESCC at $312, close to the correct $307. Its main error was using the old 20%-of-federal-CDCC care credit ($120) instead of the 606(c-2) formula, about 29.35% × $3,000 = $880.55." +us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Found no refundable state credits and missed the Empire State Child Credit ($1,000 under-4 base less a $693 phase-out = $307) and the refundable 606(c-2) child care credit ($880.55)." +us,scenario_082,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Tied the ESCC to a federal CTC it believed was fully phased out. The under-4 ESCC is a flat $1,000 with its own $16.50-per-$1,000 phase-out above $75,000, which leaves $307. It also omitted the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Fell back to the $100 ESCC minimum instead of the $1,000 under-4 base less a $693 phase-out ($307). It also computed the care credit as 20% of the $600 federal credit ($120) instead of $880.55 under 606(c-2)." +us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Used a $660 ESCC (33% of a $2,000 federal CTC) instead of the $1,000 under-4 amount reduced by $16.50 per $1,000 above $75,000 ($307). It also used the old 20%-of-federal care-credit rule ($120) instead of $880.55." +us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Looked only at the NY EITC and missed the refundable Empire State Child Credit ($307 after phase-out) and the refundable 606(c-2) child and dependent care credit ($880.55). +us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,Treated the NY child care credit as fully absorbed by tax liability. The 606(c-2) credit is refundable and counts in full ($880.55). It also omitted the ESCC of $307. +us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so there was nothing to score against the $1,187.55 reference." +us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Applied the legacy 2017-law-CTC recomputation and its $100 minimum instead of the enhanced $1,000 under-4 ESCC less $16.50 per $1,000 above $75,000 ($307). It also used 20% of the federal $600 ($120) instead of the 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Concluded that income exceeded the ESCC range, but the $1,000 under-4 credit only loses $693 at $117.7k AGI ($307 remains). It also ignored the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Checked only the NY EITC and missed the refundable Empire State Child Credit ($307 after phase-out of the $1,000 under-4 base) and the 606(c-2) child and dependent care credit ($880.55)." +us,scenario_082,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Correctly started from the $1,000 under-4 ESCC and the $75,000 threshold but used a 5% phase-out rate instead of $16.50 per $1,000 (1.65%), so the credit went to zero instead of $307. It also omitted the refundable 606(c-2) care credit of $880.55." +us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Considered only the NY EITC and concluded nothing applied, which misses the ESCC ($1,000 under-4 base less $693 = $307) and the refundable 606(c-2) child care credit ($880.55)." +us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Its $313 roughly matches the phased-down ESCC ($307), but it entirely left out the refundable child and dependent care credit. Under 606(c-2), that credit applies about 29.35% to $3,000 of expenses for $880.55." us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the individual provisions of the Tax Cuts and Jobs Act expired for 2026 and used an $8,328 single-filer standard deduction. The applicable 2026 standard deduction exceeds $11,010, so taxable income and the resulting 10% bracket tax are both zero." us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly determined that the age-20 childless filer fails the EITC age requirement and that no other refundable credit applies, but then submitted $540 instead of carrying the resulting $0 into the output." us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly concluded that the filer is too young for the childless EITC and has no qualifying child for refundable CTC, but its $538 output contradicts its own all-zero component calculation." @@ -7115,40 +7204,41 @@ us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eli us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability and blindness as sufficient for Medicaid while leaving the required income, resource, and coverage-group tests unapplied. PolicyEngine assigns no Medicaid category because those statuses alone do not establish eligibility." us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded SSDI from Medicaid MAGI merely because it considered the benefit nontaxable. Medicaid MAGI includes Social Security benefits through its program-specific adjustment, producing 1.56 times FPL rather than approximately $8,405 and placing the head above the expansion limit." us,scenario_084,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model calculated expansion MAGI as approximately $8,405 by omitting the head's Social Security disability income. The Medicaid MAGI computation yields 1.56 times FPL, so the head exceeds North Carolina's expansion threshold and has no other qualifying category." -us,scenario_084,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model named the 24-month SSDI waiting period but treated it as met because disability is constant for the year. The months of SSDI receipt are unlisted, so they are 0. Even a full year of receipt is only 12 months, short of the 24 the disability pathway requires, and at 62 the head fails the age-65 test." -us,scenario_084,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as enough for Medicare before age 65. It skipped the requirement of 24 months of SSDI entitlement, and the unlisted months of receipt are 0. At 62, the head also fails the age-65 test." -us,scenario_084,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI rule but then concluded that disability plus SSDI receipt alone confers eligibility. The months of SSDI receipt are unlisted, so they are 0 and the waiting period is not met. The head, at 62, is also under the age-65 threshold." -us,scenario_084,head_medicare_eligible,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare entitlement and ignored the 24-month waiting period. The unlisted months of SSDI receipt are 0, so the disability pathway fails, and age 62 fails the age-65 test." -us,scenario_084,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model acknowledged the 24-month SSDI waiting period but treated constant disability status as proof it was met. The unlisted months of receipt are 0, and one year of receipt gives at most 12 months, so the head is not eligible at 62." -us,scenario_084,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model explicitly assumed established SSDI receipt to clear the 24-month waiting period. The prompt forbids inferring unlisted facts, so the months of receipt are 0. The disability pathway fails, and age 62 is below 65." -us,scenario_084,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI receipt as satisfying the Medicare disability pathway and never applied the 24-month entitlement requirement. The unlisted months of SSDI receipt are 0, so the 62-year-old head is not eligible." -us,scenario_084,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model asserted that the head has been receiving SSDI past the 24-month waiting period, but no duration is given. Unlisted inputs are 0, so the waiting period is unmet, and the 62-year-old head fails the age-65 test." -us,scenario_084,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model assumed the SSDI waiting period had already passed. The months of SSDI receipt are unlisted and therefore 0, which fails the 24-month requirement, and the head is 62, under 65." -us,scenario_084,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt at age 62 as implying at least 24 months of receipt, which is an inference the prompt prohibits. The unlisted months of SSDI receipt are 0, so the 24-month waiting period is unmet and the head is not Medicare eligible." -us,scenario_084,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the Medicare disability provision on disability status and SSDI income alone. It omitted the 24-month SSDI entitlement requirement, which fails because the unlisted months of receipt are 0, and age 62 fails the age-65 test." -us,scenario_084,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model claimed that under PolicyEngine rules, being disabled with SSDI income confers Medicare eligibility. In fact, PolicyEngine requires 24 months of SSDI receipt for under-65s, and that count is 0 here because it is unlisted." -us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI income as enough for Medicare and ignored the 24-month waiting period. The months of SSDI receipt are unlisted, so they are 0, and the 62-year-old head is not eligible." -us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model concluded that receiving SSDI income alone qualifies the head for Medicare. Under-65 eligibility requires 24 months of SSDI receipt, and the unlisted months of receipt are 0." -us,scenario_084,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as qualifying for Medicare and skipped the 24-month entitlement requirement. Months of receipt default to 0 because they are unlisted, so the 62-year-old head fails both the disability and age-65 pathways." -us,scenario_084,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated disability and SSDI income as automatic Medicare eligibility. It ignored the 24-month SSDI waiting period, which is unmet because the unlisted months of receipt are 0, and age 62 is under 65." -us,scenario_084,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt as qualifying for Medicare without applying the 24-month entitlement requirement. The unlisted months of SSDI receipt are 0, so the 62-year-old head is not eligible." -us,scenario_084,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model misstated PolicyEngine's rule as disabled plus SSDI income. The actual under-65 test requires at least 24 months of SSDI receipt, and that count is 0 here because it is unlisted." -us,scenario_084,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model counted disabled and surviving-spouse status as Medicare eligibility at 62. Surviving-spouse status creates no Medicare pathway. The disability pathway requires 24 months of SSDI receipt, and the unlisted months of receipt are 0." -us,scenario_084,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly noted that age 62 does not qualify, then treated disabled status plus SSDI income as SSDI-based Medicare entitlement. That entitlement requires 24 months of SSDI receipt, and the unlisted months of receipt are 0." -us,scenario_084,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated disability and SSDI income as enough under PolicyEngine's disability rules. PolicyEngine requires 24 months of SSDI receipt for under-65 eligibility, and the months of receipt are 0 because they are unlisted." -us,scenario_084,head_medicare_eligible,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model granted disability-based Medicare from SSDI income alone and never applied the 24-month waiting period. That period fails because the unlisted months of SSDI receipt are 0, and the head is 62." -us,scenario_084,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model applied the under-65 disability pathway without checking its 24-month SSDI entitlement requirement. The unlisted months of SSDI receipt are 0, so the head is not Medicare eligible." -us,scenario_084,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model treated being a disabled SSDI beneficiary as meeting the modeled disability pathway. That pathway requires at least 24 months of SSDI receipt, and the unlisted months of receipt are 0, so the 62-year-old head fails it." -us,scenario_084,head_medicare_eligible,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI income as PolicyEngine's disability-based Medicare classification. That classification requires 24 months of SSDI receipt, which is 0 here because it is unlisted." -us,scenario_084,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model granted disability-based Medicare from current SSDI receipt and ignored the 24-month entitlement requirement. The unlisted months of SSDI receipt are 0, and age 62 fails the age-65 test." -us,scenario_084,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied the disability-based rule using only disability status and reported SSDI income. The rule requires at least 24 months of SSDI receipt, and the unlisted months of receipt are 0." -us,scenario_084,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model asserted that the head had received SSDI for a sufficient duration, but no duration appears in the facts. Unlisted numeric inputs are 0, so the 24-month waiting period fails and the 62-year-old head is not eligible." -us,scenario_084,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model claimed the 24-month SSDI waiting period was satisfied without any stated duration of receipt. The unlisted months of receipt are 0, so the disability pathway fails, and age 62 is below 65." -us,scenario_084,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model assumed the head had served the SSDI waiting period. The months of SSDI receipt are unlisted and therefore 0, short of 24, so the 62-year-old head is not Medicare eligible." -us,scenario_084,head_medicare_eligible,grok-4.7,prompt_ambiguity,age_disability,False,"The model misstated PolicyEngine's rule as age 65 or older, or disabled. Under 65, PolicyEngine requires at least 24 months of SSDI receipt, which is 0 here because it is unlisted, so disability status alone does not confer eligibility." -us,scenario_084,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,The model treated disability plus SSDI income as qualifying for Medicare under 65 and omitted the 24-month SSDI entitlement requirement. That requirement fails because the unlisted months of receipt are 0. -us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model assumed the 24-month waiting period from the size of the disability and SSDI income amounts. Benefit amounts say nothing about duration, and the unlisted months of SSDI receipt are 0, so the 62-year-old head fails the disability pathway." -us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model read 'constant throughout the year' as satisfying the 24-month SSDI requirement. Constant status over one year gives at most 12 months, and the unlisted months of receipt are 0, so the head is not Medicare eligible at 62." +us,scenario_084,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model named the 24-month SSDI waiting period but treated a disability status that holds for the whole year as meeting it. The prompt gives no SSDI duration, so months of SSDI receipt are 0, which is under 24, and the age-62 Head is not Medicare eligible." +us,scenario_084,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model held that SSDI receipt by itself makes someone eligible for Medicare before 65 and skipped the 24-month entitlement requirement. With no SSDI duration listed, months of receipt default to 0, so the 62-year-old Head fails both the age test and the disability test." +us,scenario_084,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model stated the 24-month SSDI rule and then concluded that being disabled and receiving SSDI is enough, without checking the duration. The listed facts give 0 months of SSDI receipt, so the under-65 Head is ineligible." +us,scenario_084,head_medicare_eligible,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model treated current SSDI benefits as automatic Medicare entitlement and ignored the 24-month waiting period. With SSDI duration unlisted and so 0, the age-62 Head does not qualify." +us,scenario_084,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI waiting period but took a constant disability status within the year as meeting it. The months-receiving-SSDI input is unlisted and therefore 0, so the 62-year-old Head is not eligible." +us,scenario_084,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model said it was 'assuming established disability benefit receipt' to meet the 24-month rule, which infers a duration the prompt does not give. Months of SSDI receipt are 0, so the Head, at 62, fails the Medicare disability pathway." +us,scenario_084,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt plus disability as meeting the Medicare disability pathway and never applied the 24-month entitlement requirement. With 0 listed months of SSDI receipt, the under-65 Head is ineligible." +us,scenario_084,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model asserted that the Head 'has been receiving' SSDI past the 24-month waiting period, but no duration is given and unlisted numeric inputs are 0. With 0 months of SSDI and an age of 62, the Head is not eligible for Medicare." +us,scenario_084,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model treated the SSDI waiting period as already met without any stated duration. Months of SSDI receipt default to 0, which is below the 24-month threshold, so the 62-year-old Head is ineligible." +us,scenario_084,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model inferred from the Head's age of 62 that SSDI had been received for at least 24 months, which the prompt does not support. Unlisted numeric inputs are 0, so months of SSDI receipt are 0 and the Head fails the disability pathway." +us,scenario_084,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the Medicare disability provision on the basis of SSDI receipt alone and left out the 24-month entitlement requirement. With 0 listed months of SSDI, the under-65 Head does not qualify." +us,scenario_084,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model claimed that PolicyEngine grants Medicare for disability plus SSDI income, but PolicyEngine requires at least 24 months of SSDI receipt. That input is unlisted and therefore 0, so the 62-year-old Head is ineligible." +us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model concluded that being disabled and receiving SSDI makes someone eligible for Medicare and never applied the 24-month SSDI duration test. Months of receipt are 0, so the Head, at 62, is not eligible." +us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model treated SSDI income alone as qualifying the Head for Medicare and skipped the 24-month waiting period. With SSDI duration unlisted and so 0, the under-65 Head is ineligible." +us,scenario_084,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model equated current SSDI benefits with Medicare eligibility and ignored the requirement of at least 24 months of SSDI entitlement. Months of receipt are 0, so the 62-year-old Head does not qualify." +us,scenario_084,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated disability plus SSDI income as enough for Medicare and never checked the 24-month SSDI duration. That duration is unlisted and therefore 0, so the Head, at 62, is ineligible." +us,scenario_084,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model concluded that SSDI receipt qualifies the Head for Medicare and left out the 24-month waiting period. With 0 months of SSDI receipt and an age under 65, the Head is not eligible." +us,scenario_084,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's rules grant Medicare to anyone under 65 who is disabled and receiving SSDI. PolicyEngine actually requires at least 24 months of SSDI receipt, and that input is 0 here, so the Head is ineligible." +us,scenario_084,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model treated disability and surviving-spouse status as indicating Medicare eligibility, but surviving-spouse status has no bearing on it. The disability pathway requires 24 months of SSDI receipt, which is 0 here, so the 62-year-old Head is ineligible." +us,scenario_084,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly noted that Medicare is not automatic at 62 but then counted disabled status plus SSDI income as SSDI-based Medicare entitlement. PolicyEngine requires at least 24 months of SSDI receipt, and that input defaults to 0, so the Head is not eligible." +us,scenario_084,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model said disability plus SSDI income supports Medicare eligibility under PolicyEngine's disability rules and skipped the 24-month SSDI duration threshold. With 0 months of SSDI receipt, the under-65 Head fails that test." +us,scenario_084,head_medicare_eligible,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt alone as qualifying the Head for Medicare and never applied the 24-month entitlement requirement. Months of receipt are unlisted and therefore 0, so the 62-year-old Head is ineligible." +us,scenario_084,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model applied the Medicare disability pathway based on disability status and SSDI income without the 24-month waiting-period test. With 0 months of SSDI receipt, the age-62 Head does not qualify." +us,scenario_084,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's disability-based Medicare pathway applies to any disabled SSDI beneficiary. That pathway requires at least 24 months of SSDI receipt, and the input is 0 here, so the Head is ineligible." +us,scenario_084,head_medicare_eligible,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model classified the Head as eligible for disability-based Medicare because of disability status plus SSDI income and skipped the 24-month SSDI duration requirement. Months of receipt are 0, so the under-65 Head fails it." +us,scenario_084,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model treated a disabled SSDI recipient as automatically eligible for disability-based Medicare and ignored the 24-month waiting period. That period is unmet because months of SSDI receipt default to 0, so the 62-year-old Head is ineligible." +us,scenario_084,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied the disability-based Medicare rule on the basis of reported disability and SSDI income, but that rule also requires at least 24 months of SSDI receipt. That input is unlisted and therefore 0, so the Head, at 62, is not eligible." +us,scenario_084,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model asserted that the Head had been on SSDI 'for sufficient duration', a fact the prompt never gives. Months of SSDI receipt default to 0, below the 24-month threshold, so the under-65 Head is ineligible." +us,scenario_084,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model claimed that disability plus SSDI income meets the 24-month waiting-period rule, but no duration is listed and unlisted numeric inputs are 0. With 0 months of SSDI receipt, the 62-year-old Head is not Medicare eligible." +us,scenario_084,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model named the SSDI waiting period but treated it as met without any stated duration of receipt. Months of SSDI receipt are 0, so the Head, at 62, fails the 24-month test and is ineligible." +us,scenario_084,head_medicare_eligible,grok-4.7,prompt_ambiguity,age_disability,False,"The model stated PolicyEngine's rule as 'age 65 or older or disabled'. The rule actually requires age 65+, ESRD, or at least 24 months of SSDI receipt. Disability status alone does not qualify, and months of SSDI receipt are 0, so the Head is ineligible." +us,scenario_084,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt by an under-65 disabled person as qualifying for Medicare and never applied the 24-month entitlement requirement. Months of receipt are unlisted and therefore 0, so the Head is ineligible." +us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model assumed the 24-month waiting period was met because disability benefit amounts were present, confusing the dollar amount of benefits with how long they had been received. The duration is unlisted and therefore 0, so the 62-year-old Head is not eligible." +us,scenario_084,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disability,False,"The model granted Medicare on disability grounds because of SSDI receipt and skipped the 24-month SSDI entitlement threshold. With 0 months of SSDI receipt and an age of 62, the Head does not qualify." +us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model stated the 24-month SSDI rule and then took a constant disability status within the year as meeting it, which confuses a status held all year with 24 months of entitlement. Months of SSDI receipt are unlisted and therefore 0, so the Head is ineligible." us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Sonnet derived monthly net income of $1,737.59 but never compared it to the SNAP net income limit of 100% FPL (about $1,340/month for one person), which that figure fails by roughly 30%, and it ignored both NC's 200%-FPL BBCE gross cap (about $2,680/month against its own $2,769) and the $56,500 in bank assets against the elderly/disabled resource limit of roughly $4,650. Its own benefit formula returned $0 because 30% of net income ($521.28) exceeds the one-person maximum allotment ($298), after which it discarded that result and substituted an unsupported $183/month figure attributed to a conditional minimum benefit, a rule that provides $23-$24, not $183. It further inflated the excess medical deduction by double-counting the same $3,600 premium (listed once as health insurance premiums excluding Medicare Part B and again as other health insurance premiums) and by including $1,054 of non-prescription over-the-counter costs, which are not deductible medical expenses." us,scenario_084,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"DeepSeek asserted the household passes the net income test after the standard and medical deductions, but net income of about $1,738/month exceeds the one-person net income limit near $1,340/month, gross income of $2,769/month exceeds North Carolina's 200%-FPL BBCE gross cap of about $2,680/month, and $56,500 in countable resources exceeds the elderly/disabled asset limit of roughly $4,650. Having wrongly cleared eligibility, it applied the $23/month minimum allotment for one- and two-person households, a floor that attaches only after a household qualifies; an income-ineligible household receives $0, not $276." us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got every step right: taxable SS $14,051, AGI $45,574, deductions of $18,150 + $6,000, and tax of about $2,323. It then submitted $940, which contradicts its own calculation and has no basis in the reasoning." @@ -7192,89 +7282,91 @@ us,scenario_085,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correc us,scenario_085,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_085,snap,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_085,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It correctly got PA taxable income down to the $1,675 of interest and $51.42 of tax. It then ruled out Tax Forgiveness by adding the $27,426 of Social Security and the pensions to eligibility income. Schedule SP eligibility income leaves out both, so it is $1,675, under the $6,500 single threshold, which gives 100% forgiveness and $0 tax." -us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It computed $51.42 on the $1,675 of interest. It then denied Tax Forgiveness by treating Social Security and exempt retirement distributions as eligibility income. Schedule SP leaves both out, so eligibility income is $1,675, below the $6,500 single limit, and full forgiveness brings the tax to $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built PA taxable income from federal AGI including Social Security and all retirement distributions, and applied a made-up $8,700 PA standard deduction. It also used a 3.077% rate. PA exempts Social Security and a 67-year-old's pension, IRA and 401(k) distributions and has no standard deduction, so taxable income is only $1,675 of interest. Schedule SP Tax Forgiveness then cancels that tax in full, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It stopped at 3.07% tax on the $1,675 of interest and never applied PA Special Tax Forgiveness. Eligibility income is $1,675, below the $6,500 single threshold, so forgiveness is 100% and tax is $0. It also misrounded $51.4225 to $51.45." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly limited PA taxable income to the $1,675 of interest but never applied Schedule SP Tax Forgiveness. Eligibility income excludes Social Security and retirement benefits, so it is $1,675, under the $6,500 single limit. Full forgiveness wipes out the $51.42, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It assumed forgiveness eligibility income includes the exempt retirement income and so exceeds the forgiveness limit. Schedule SP eligibility income leaves out commonly recognized retirement benefits and Social Security, so it equals the $1,675 of interest. That is below $6,500, so the $51.42 is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded that no PA nonrefundable credits apply and overlooked Special Tax Forgiveness. Eligibility income of $1,675 (interest only) is below the $6,500 single threshold, so 100% forgiveness reduces the tax to $0. It submitted $51, the tax before forgiveness, rounded." -us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own reasoning reached $51 on the $1,675 of interest, but it submitted an arbitrary $500 'to reflect potential additional taxable classes', which contradicts that derivation. It also never applied Schedule SP Tax Forgiveness, which fully forgives tax when eligibility income ($1,675) is below $6,500. The correct result is $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It taxed the $1,675 of interest at 3.07% and stopped, never applying PA Special Tax Forgiveness. Eligibility income excludes Social Security and exempt retirement distributions, so it is $1,675, under the $6,500 single limit. Full forgiveness gives $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made up a $1,250 senior interest exclusion that does not exist in PA law, cutting taxable interest to $425. It also said no state credits apply, missing Tax Forgiveness. With eligibility income of $1,675, well under the $6,500 single threshold, Schedule SP forgives 100% of the tax, so the answer is $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It declared that no PA nonrefundable credits apply and overlooked Special Tax Forgiveness. Eligibility income is only the $1,675 of interest, since Social Security and retirement benefits are excluded. That is below $6,500, so the $51 is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single-filer threshold, so 100% forgiveness gives $0 tax." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It correctly exempted Social Security and retirement distributions but stopped at $51.42 without applying Schedule SP Tax Forgiveness. Eligibility income is $1,675, below $6,500, so the tax is 100% forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"$1,782 is about 3.07% of roughly $58,000, which is the household's full $58,949 of gross income. So despite claiming to exclude Social Security, it taxed essentially all income, including pension, IRA and 401(k) distributions that PA exempts after age 59½. Correctly, taxable income is $1,675 of interest, and Tax Forgiveness then eliminates that tax, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It taxed only the $1,675 of interest but left out PA Special Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single threshold, so full forgiveness reduces the $51 to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It reported $51.42 as final tax without applying Schedule SP Tax Forgiveness. Social Security and retirement benefits are excluded from eligibility income, so eligibility income is $1,675, under the $6,500 single limit, and the tax is fully forgiven to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It stopped at 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Eligibility income of $1,675 is below $6,500, which gives 100% forgiveness and $0 tax." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It correctly identified $1,675 as the only PA-taxable income but skipped Special Tax Forgiveness. That same $1,675 is the Schedule SP eligibility income, below the $6,500 single threshold, so the $51.42 is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It computed $51.42 on the interest and treated that as the final tax, never applying PA Tax Forgiveness. Eligibility income excludes Social Security and exempt retirement distributions, so it is $1,675, below $6,500, and full forgiveness gives $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It explicitly applied no nonrefundable credits and so missed Schedule SP Special Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single limit, so 100% forgiveness reduces tax to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed the $1,675 of interest at 3.07% and stopped without applying PA Tax Forgiveness. Eligibility income is $1,675, below the $6,500 single threshold, so the tax is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It correctly excluded retirement distributions and Social Security but never applied Special Tax Forgiveness. Eligibility income of $1,675 is under $6,500, so 100% forgiveness gives $0 instead of $51.42." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It ruled out Tax Forgiveness on the basis of total household income, counting Social Security and retirement distributions. Schedule SP eligibility income leaves those out, so it is only the $1,675 of interest, below the $6,500 single limit, and full forgiveness brings the tax to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It stopped at about $51 of tax on the interest without applying PA Special Tax Forgiveness. Eligibility income of $1,675 is below the $6,500 single threshold, so the tax is fully forgiven to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It computed 3.07% of the $1,675 of interest and never applied Schedule SP Tax Forgiveness. Eligibility income of $1,675 is under $6,500, which gives 100% forgiveness and $0 tax." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It concluded that no nonrefundable credit applies and overlooked PA Special Tax Forgiveness. Eligibility income excludes Social Security and retirement benefits, so it is $1,675, below the $6,500 single limit. Full forgiveness wipes out the $51.42, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It applied the flat 3.07% rate to the $1,675 of interest and stopped, never applying PA Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single threshold, so the tax is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It included the $20,720 pension and the $7,568 IRA and $1,560 401(k) distributions in PA taxable income. PA exempts these retirement distributions for a retiree aged 59½ or older. Correct taxable income is $1,675 of interest, and Schedule SP forgiveness then eliminates that tax because eligibility income is below $6,500, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It taxed only the $1,675 of interest but never applied Special Tax Forgiveness. Eligibility income of $1,675 is below $6,500, so 100% forgiveness reduces the $51 to $0." -us,scenario_085,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It stopped at 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Social Security and retirement benefits are excluded from eligibility income, so it is $1,675, under the $6,500 single threshold, and full forgiveness gives $0." -us,scenario_085,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It taxed the $29,848 of pension, IRA and 401(k) distributions, which PA excludes for a 67-year-old retiree, reaching $31,523 of taxable income. On that inflated base it then rejected Tax Forgiveness. The correct base is $1,675 of interest, which also serves as eligibility income; that is below $6,500, so forgiveness is 100% and tax is $0." -us,scenario_085,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It correctly reduced PA taxable income to the $1,675 of interest but asserted that no nonrefundable credits apply, missing Special Tax Forgiveness. Eligibility income of $1,675 is below the $6,500 single limit, so the $51.42 is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It rejected Tax Forgiveness by treating Social Security and exempt retirement distributions as eligibility income. Schedule SP leaves out both, so eligibility income is only $1,675, under the $6,500 single threshold. The $51.42 is fully forgiven, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the pension, IRA and 401(k) distributions as PA-taxable, giving $31,523 of taxable income. PA exempts retirement distributions received after age 59½/retirement, so taxable income is only the $1,675 of interest. It also missed that Schedule SP Tax Forgiveness zeroes that tax because eligibility income is below $6,500, so the answer is $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model correctly reduced PA taxable income to the $1,675 of interest. It then denied Tax Forgiveness by counting Social Security and pensions in eligibility income, but Schedule SP leaves those out. Eligibility income of about $1,675 is under the $6,500 single limit, so 100% forgiveness wipes out the $51.42 and the answer is $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model taxed only the $1,675 of interest but said eligibility income was far above the forgiveness limits. PA eligibility income leaves out Social Security and post-retirement-age pension, IRA and 401(k) distributions, so it is about $1,675, under the $6,500 threshold. Full Tax Forgiveness cuts the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model put Social Security and retirement distributions into PA taxable income, though PA exempts both for a 67-year-old. It also subtracted an $8,700 standard deduction that PA does not have. PA taxable income is only the $1,675 of interest, and 100% Tax Forgiveness removes that liability, so the answer is $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model stopped at 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Eligibility income leaves out Social Security and retirement distributions, so it is about $1,675, below the $6,500 single limit for full forgiveness. The tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model correctly exempted Social Security and retirement distributions and taxed $1,675 of interest, but it left out the nonrefundable PA Tax Forgiveness credit. With eligibility income of about $1,675, under the $6,500 single threshold, forgiveness is 100% and the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model assumed retirement income counts toward forgiveness eligibility income and so denied forgiveness. Schedule SP eligibility income leaves out Social Security and age-qualified retirement distributions, leaving about $1,675, under the $6,500 limit. Full Tax Forgiveness cuts the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said no PA nonrefundable credits apply and left $51 of tax on the $1,675 of interest. It missed PA Tax Forgiveness, which is 100% here because eligibility income (no Social Security, no retirement distributions) is about $1,675, below $6,500. The tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model's reasoning arrived at $51 on the interest but it submitted an unsupported $500. It also never applied PA Tax Forgiveness. Eligibility income of about $1,675 qualifies for 100% forgiveness, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model correctly limited PA taxable income to $1,675 of interest but stopped at $51 of gross tax. It skipped the PA Tax Forgiveness credit, which is 100% because eligibility income of about $1,675 is under the $6,500 single limit. The tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model invented a $1,250 interest exclusion for people age 59½ and over, which PA does not have, and taxed $425. It also said no nonrefundable credits apply, missing PA Tax Forgiveness. Eligibility income of about $1,675 gets 100% forgiveness, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said no PA nonrefundable credits apply and kept $51 of tax on the interest. PA Tax Forgiveness is a nonrefundable credit. With Social Security and retirement distributions left out, eligibility income is about $1,675, under $6,500, so full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model computed 3.07% of the $1,675 of interest and never checked PA Tax Forgiveness. Eligibility income, which leaves out Social Security and age-qualified retirement distributions, is under the $6,500 single threshold, so 100% forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly exempted retirement distributions and Social Security but stopped at $51.42 on the interest. It never applied PA Tax Forgiveness, which fully forgives the tax because eligibility income is about $1,675, under $6,500. The tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The $1,782 answer equals 3.07% of about $58,000, so the model taxed nearly all household income. That base includes retirement distributions, which PA exempts for a 67-year-old. PA taxable income is only $1,675 of interest, and 100% Tax Forgiveness cuts that liability to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model limited taxable income to $1,675 of interest but did not apply PA Tax Forgiveness. Eligibility income of about $1,675, with Social Security and retirement income left out, is below the $6,500 single limit. Full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stopped at $51.42 of gross PA tax on the interest and skipped the nonrefundable PA Tax Forgiveness credit. Eligibility income is about $1,675, under the $6,500 threshold, so the credit removes the whole liability and the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model taxed the $1,675 of interest at 3.07% and did not apply PA Tax Forgiveness. Eligibility income leaves out Social Security and retirement distributions and falls below $6,500, so 100% forgiveness applies and the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model correctly isolated $1,675 of interest income but never applied PA Tax Forgiveness. Eligibility income of about $1,675 qualifies for 100% forgiveness under the $6,500 single limit, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model correctly exempted Social Security and retirement distributions but stopped at $51.42. It missed the PA Tax Forgiveness credit. Eligibility income of about $1,675 falls under the $6,500 single limit and gets full forgiveness, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model said no nonrefundable credits apply and left $51 on the interest. PA Tax Forgiveness is a nonrefundable credit that fully forgives the tax, because eligibility income (no Social Security, no retirement distributions) is about $1,675, under $6,500. The tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model taxed $1,675 of interest at 3.07% and never applied PA Tax Forgiveness. With eligibility income near $1,675, below the $6,500 single limit, 100% forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model correctly excluded retirement distributions and Social Security but did not apply the PA Tax Forgiveness credit. Eligibility income of about $1,675 is under the $6,500 threshold, so the tax is fully forgiven to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model said total household income rules out tax forgiveness, but Schedule SP eligibility income leaves out Social Security and age-qualified retirement distributions. Eligibility income is about $1,675, under $6,500, so 100% forgiveness cuts the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model computed about $51 on the interest and skipped PA Tax Forgiveness. Eligibility income, with Social Security and retirement distributions left out, is about $1,675, below the $6,500 single limit. Full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model stopped at 3.07% of $1,675 of interest and did not apply PA Tax Forgiveness. Eligibility income near $1,675 qualifies for 100% forgiveness, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model said no nonrefundable credit applies, but PA Tax Forgiveness does. Eligibility income leaves out Social Security and retirement distributions, so it is about $1,675, under the $6,500 single threshold. That gives 100% forgiveness and $0 of tax." +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model applied the flat 3.07% rate to the $1,675 of interest and did not apply PA Tax Forgiveness. Eligibility income of about $1,675 is below $6,500, so full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model taxed the $20,720 pension and the IRA and 401(k) distributions, but PA exempts retirement distributions received after retirement age by a 67-year-old. It also said no nonrefundable credits apply. With only $1,675 of interest taxable and eligibility income under $6,500, Tax Forgiveness cuts the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model correctly limited the base to $1,675 of interest but stopped at $51 of gross tax. It skipped PA Tax Forgiveness, which fully forgives liability when eligibility income is under $6,500. The tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model taxed only the $1,675 of interest but missed the nonrefundable PA Tax Forgiveness credit. Eligibility income of about $1,675, with Social Security and retirement income left out, gets 100% forgiveness, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model taxed $31,523 including pension, IRA and 401(k) distributions, but PA exempts these for a retiree over age 59½. It then denied forgiveness on that inflated income. Correctly, taxable and eligibility income are both about $1,675, which qualifies for 100% Tax Forgiveness and $0 of tax." +us,scenario_085,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model correctly reduced PA taxable income to $1,675 of interest, then said no nonrefundable credits apply. PA Tax Forgiveness applies at 100% because eligibility income is about $1,675, below the $6,500 single limit, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model said forgiveness is unavailable because eligibility income is much higher, which means it counted Social Security and retirement distributions. Schedule SP eligibility income leaves those out, so it is about $1,675, under $6,500. Full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"The model taxed the $1,675 of interest at 3.07% and never applied PA Tax Forgiveness. Eligibility income of about $1,675 qualifies for 100% forgiveness under the $6,500 single threshold, so the tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model put the pension, IRA and 401(k) distributions ($29,848) into PA taxable income, but PA exempts retirement distributions received after retirement age by a 67-year-old. It also said no nonrefundable credits apply. Only $1,675 of interest is taxable, and 100% Tax Forgiveness cuts the tax to $0." us,scenario_086,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated Medicaid ineligibility as sufficient for CHIP eligibility and never applied Georgia’s CHIP income limit. Although the 12-year-old satisfies the age criterion, the household fails the CHIP income criterion, so the child is not eligible." us,scenario_086,child1_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly found that income exceeds Georgia’s child Medicaid limit, then reversed its result by treating PeachCare/CHIP as part of PolicyEngine’s Medicaid output. Medicaid and CHIP are separate benchmark variables, and child1 qualifies for no Medicaid category at 3.44 times FPL." us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the head’s employer-sponsored insurance premiums from Medicaid MAGI and then asserted that $74,824 was within a roughly $50,487 limit. Those premiums do not reduce MAGI on the stated facts, and 3.44 times FPL exceeds Georgia’s child Medicaid thresholds." us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model invented a three-person household despite the prompt listing only the head and child, then conflated the 247% FPL PeachCare/CHIP ceiling with Medicaid eligibility. For the actual two-person household, child1 is at 3.44 times FPL and has no Medicaid eligibility category." us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model relied on age-based categorical eligibility without applying Georgia’s income threshold for a 12-year-old child. Child1’s MAGI income level is 3.44 times FPL, above the applicable child Medicaid limits, leaving the Medicaid category as NONE." us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Treated the Social Security dependent benefits as the child's income, so it left the $10,900.40 taxable portion out of Head's AGI. It also deducted $275 of auto-loan interest even though the facts show no qualifying vehicle loan. It then submitted $3,305, which contradicts its own $1,925 result." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion, the $24,150 HoH deduction and the $2,200 CTC right. However, it subtracted $275 of auto-loan interest (worth $33 of tax) that the facts don't support. It also ended the 10% bracket at $17,000 instead of $17,700, adding $14 and leaving a net shortfall of $19.06." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Filed as single with a $14,600 deduction instead of HoH with $24,150. It included the full $12,824 of benefits rather than the 85% taxable share, and subtracted employer and other health premiums as above-the-line deductions. It then submitted $3,779 without subtracting any CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Used married-filing-jointly treatment for a single parent who should file as head of household. It also built taxable income from wages alone, leaving out the $10,900.40 of taxable Social Security." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Treated the Social Security dependent benefits as the child's and left the $10,900.40 taxable portion out of AGI. It also used a $23,000 HoH deduction instead of $24,150, a 10% bracket ending at $17,000 instead of $17,700, and a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Left the $10,900.40 of taxable Social Security out of AGI by assigning the benefits to the child, and used a $23,000 HoH deduction instead of $24,150. It then submitted $4,673, an unexplained pre-credit figure, rather than its own $2,673 after the CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Said the dependent benefits weren't taxable at this income, but provisional income of about $68,075 is above the $34,000 base, so $10,900.40 is taxable. It also used a $24,500 deduction instead of $24,150 and a $2,000 CTC instead of $2,200, then inflated its own $4,119 tax figure to $4,519 with no explanation." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Capped the educator expense deduction at $300, while the reference allows $337.50 above the line. That $37.50 of extra AGI, taxed at 12%, is the whole $4.50 gap; its 85% Social Security inclusion, $24,150 HoH deduction, brackets and $2,200 CTC all match the reference." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA expired in 2026 and applied pre-TCJA rules: a single-filer deduction, personal exemptions, a 15% bracket and a $1,000 CTC. Under the OBBBA-extended rules, a HoH filer gets $24,150, no exemptions, a 10%/12% schedule and a $2,200 CTC. It also left the $10,900.40 of taxable Social Security out by treating the benefits as the child's." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Treated the Social Security benefits as non-taxable, leaving out the $10,900.40 that the 85% rule includes. It used outdated figures ($23,350 deduction, 10% bracket ending at $16,550, $2,000 CTC), then replaced its own $2,271 result with an unexplained $3,892." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Matched the reference except that it subtracted $275 of auto-loan interest. The facts don't establish a qualified new, U.S.-assembled vehicle loan, so that deduction doesn't apply, and it cut tax by $33." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Included the taxable Social Security correctly but used a $23,100 HoH deduction instead of the 2026 figure of $24,150. It also used a $2,000 CTC instead of $2,200 and rounded the bracket tax roughly." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA rules ($11,000 deduction, $9,600 of exemptions, 15% bracket, $1,000 CTC) instead of the 2026 OBBBA rules. It subtracted the $1,560 health premium from wages, and it never included the $10,900.40 of taxable Social Security." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion right but used a $23,100 HoH deduction instead of $24,150, an older 10% bracket limit instead of $17,700, and a $2,000 CTC instead of the 2026 amount of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: a $12,200 deduction, $10,600 of personal exemptions, a 15% bracket and a $1,000 CTC. The OBBBA-extended law gives a $24,150 HoH deduction, no exemptions, a 10%/12% schedule and a $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $19,648 employer-sponsored insurance premiums from the $62,000 of gross wages, although the full $62,000 counts as employment income. It also applied TCJA-sunset rules (personal exemptions, 15% bracket, $1,000 CTC) instead of the 2026 HoH $24,150 deduction and $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Put the full $12,824 of benefits into gross income instead of the 85% taxable share of $10,900.40, used a $15,300 deduction instead of the HoH $24,150, and used a $2,000 CTC. The $1,782 doesn't even follow from those inputs." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Got AGI right at about $72,560 but assumed the TCJA expired in 2026. It applied personal exemptions, a pre-TCJA HoH deduction, a 15% bracket and a $1,000 CTC, instead of the $24,150 deduction, 10%/12% schedule and $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $19,648 employer-sponsored premiums from the gross wages, and applied TCJA-sunset rules ($15,000 deduction, $10,100 of exemptions, $1,000 CTC) instead of the 2026 HoH $24,150 deduction and $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation. The correct path ($10,900.40 taxable SS, $72,562.90 AGI, $24,150 HoH deduction, $5,455.55 tax less $2,200 CTC) gives $3,255.55. Its $3,043 is $212.55 lower, which matches understating taxable income by about $1,771 at the 12% rate." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Applied a $2,000 CTC instead of $2,200. Its $3,678 matches a pre-credit tax of about $5,678, which comes from using the old $22,500 HoH deduction and a $17,000 10% bracket instead of the 2026 figures of $24,150 and $17,700." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Subtracted personal exemptions, which are zero under the OBBBA-permanent TCJA rules, on top of the HoH standard deduction. That cut taxable income well below $48,412.90 and left it $1,366.81 short." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion right but used the 2025 pre-OBBBA $22,500 HoH deduction instead of $24,150, a 10% bracket ending at $17,000 instead of $17,700, and a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Treated the Social Security dependent benefits as the child's and left the $10,900.40 taxable portion out of Head's AGI. It also used a $22,500 HoH deduction instead of $24,150 and a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Got AGI ($72,562) and the $2,200 CTC right, but the bracket math was wrong. It computed $6,347 of tax on about $48,212, when the 10%/12% HoH schedule gives about $5,431; it also used a $24,350 deduction instead of $24,150." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the CTC and other nonrefundable credits wipe out the liability. In fact the $5,455.55 of pre-credit tax is far more than the only nonrefundable credit, the $2,200 CTC, so $3,255.55 of tax remains." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Built taxable income from wages alone, leaving out the $10,900.40 of taxable Social Security. It also subtracted medical expenses on top of the standard deduction, although the $3,060 total is below the 7.5%-of-AGI floor and can't be combined with the standard deduction." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Left the $10,900.40 of taxable Social Security out and used a $22,500 deduction instead of $24,150. It then cut the nonrefundable CTC to about $384 based on a supposed refundable portion, but the $2,200 CTC applies in full as a nonrefundable credit because tax exceeds it." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Limited taxable Social Security to 50% ($6,412). Provisional income of about $68,075 exceeds the $34,000 adjusted base, so the 85% tier applies and $10,900.40 is taxable." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Got the income and credit right, but its $3,269 is $14 above the reference. That gap equals ending the 10% HoH bracket at the 2025 limit of $17,000 instead of the 2026 limit of $17,700." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct $24,150 HoH deduction, 2026 brackets and $2,200 CTC, but built taxable income from wages only. It left out the $10,900.40 of Social Security that the 85% rule makes taxable for Head." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Left Social Security out of income and subtracted mortgage interest that the facts don't list (only a loan balance is given). With wage-only income its own method gives about $1,950, so the $4,523 also fails its own arithmetic." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Got AGI right at $72,562 but applied TCJA-sunset rules: a $12,389 deduction, $10,733 of exemptions, a 15% bracket and a $1,000 CTC. It should have used the 2026 HoH $24,150 deduction, the 10%/12% schedule and the $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Got the 85% Social Security inclusion right but applied TCJA-sunset rules ($12,400 deduction, two $5,350 exemptions, 15% bracket, $1,000 CTC) instead of the 2026 OBBBA HoH $24,150 deduction and $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Computed $5,464 of tax on $48,440 of taxable income, when the 2026 HoH schedule gives $5,458.80. It also took 85% of the benefits as $10,890 instead of $10,900.40 and capped the educator deduction at $300 rather than the reference's $337.50." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Got AGI right but applied TCJA-sunset rules ($12,538 deduction, $10,862 of exemptions, 15% bracket, $1,000 CTC) instead of the 2026 HoH $24,150 deduction, 10%/12% schedule and $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Capped the educator expense deduction at $300, while the reference allows $337.50, which raises tax by $4.50. Taking 85% of the benefits as $10,890 instead of $10,900.40 was a small offsetting slip; its HoH deduction, brackets and $2,200 CTC match." -us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Returned no value and no explanation for federal_income_tax_before_refundable_credits. -us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Subtracted $275 of auto-loan interest (as an above-the-line item). The facts don't establish a qualified new, U.S.-assembled vehicle loan, so no such deduction applies, and the extra deduction cut tax by $33; everything else matches." -us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Filed as single with a $15,000 deduction and single brackets instead of head of household with a qualifying child ($24,150 deduction, 10% bracket up to $17,700). It also left out the $10,900.40 of taxable Social Security and used a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Filed as single ($16,100 deduction, 22% bracket) instead of HoH and deducted $275 of auto-loan interest that doesn't apply. It also allowed only $500 of nonrefundable CTC, but the full $2,200 is used against tax first because liability exceeds it." -us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Treated a parent and child as married filing jointly and applied MFJ Social Security thresholds and a $31,200 deduction. The correct status is head of household with a $24,150 deduction; it also used a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Applied the $32,200 MFJ deduction to a single parent and claimed the Social Security benefits are excluded, when 85% ($10,900.40) is taxable. It then reported zero tax even though $74,824 minus $32,200 is still positive." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $12,824 of Social Security dependent benefits as the child's income and left out the $10,900.40 (85%) taxed on the head's return. It also subtracted $275 of auto loan interest, which is not deducted here. Its own steps reached $1,925 ($4,125 tax less the $2,200 CTC), yet it submitted an unrelated $3,305." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got the 85% taxable Social Security and the $24,150 HoH standard deduction right, but also subtracted $275 of auto loan interest, which is not allowed here. That put taxable income at $48,137.40 instead of $48,412.90. It also ended the 2026 HoH 10% bracket at $17,000 instead of $17,700, which only partly offsets the deduction error." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $19,648 of employer-sponsored premiums and both $1,560 health premium entries as above-the-line deductions, which they are not, and counted all $12,824 of benefits instead of the $10,900.40 taxable portion. It also filed single with the 2024 $14,600 deduction instead of HoH with $24,150, used a $2,000 CTC instead of $2,200, and submitted its $3,779 tax before the credit." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It computed the tax as a married-filing-jointly household, but a single parent with a 12-year-old files head of household. It also left out the $10,900.40 of taxable Social Security. The correct computation taxes $48,412.90 of HoH taxable income for $5,455.55, then subtracts the $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as the child's non-taxable income and left the $10,900.40 taxable portion out of the head's AGI. It also used an outdated $23,000 HoH standard deduction instead of OBBBA's $24,150 and a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assigned the benefits to the child, leaving out the $10,900.40 of taxable Social Security, and used a $23,000 HoH standard deduction instead of $24,150. It then swapped its computed $4,304 tentative tax for an unexplained $4,673 and submitted that before-credit figure. It never subtracted the CTC, even though its text shows $2,673." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It said the Social Security benefits are not taxable at this income, but provisional income of about $68,075 is far above the $34,000 upper base, so 85% ($10,900.40) is taxable. It also used a $24,500 standard deduction instead of $24,150, raised its own $4,119 bracket result to $4,519, and subtracted a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its computation matches the reference at every step except the educator expense. It capped that deduction at $300, while the reference deducts the full $337.50, which leaves $37.50 more taxable income ($48,450.40) and $4.50 more tax ($3,260.05)." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expired after 2025 and used pre-TCJA rules: personal exemptions, an $8,300 single standard deduction, a 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has a $24,150 HoH deduction, 10%/12% brackets and a $2,200 CTC. It also filed single instead of head of household and left out the $10,900.40 of taxable Social Security." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $12,824 of Social Security benefits as non-taxable even though provisional income is far above $34,000, which makes 85% ($10,900.40) taxable. It also used a $23,350 standard deduction instead of $24,150 and a $2,000 CTC instead of $2,200. It computed $2,271 and then submitted an unexplained $3,892." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its taxable Social Security, educator deduction, standard deduction, brackets and $2,200 CTC all match. But it also subtracted $275 of auto loan interest as a qualified vehicle-loan interest deduction, which is not allowed here, cutting taxable income to $48,137 and the tax by $33." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $23,100 HoH standard deduction instead of OBBBA's 2026 amount of $24,150 and a $2,000 CTC instead of $2,200. It also capped educator expenses at $300 instead of deducting $337.50. Together these overstate taxable income and the after-credit tax." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rules ($11,000 standard deduction, $9,600 of personal exemptions, a 15% bracket, $1,000 CTC) instead of OBBBA's permanent $24,150 HoH deduction, 10%/12% brackets and $2,200 CTC. It also subtracted $1,560 of health premiums from wages and left out the $10,900.40 of taxable Social Security." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It got the 85% taxable Social Security right but used a $23,100 HoH standard deduction instead of OBBBA's 2026 $24,150 and subtracted a $2,000 CTC instead of $2,200. It also capped educator expenses at $300 instead of deducting $337.50." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expired, using a $12,200 standard deduction, $10,600 for two personal exemptions, a 15% bracket and a $1,000 CTC. Under OBBBA, 2026 has a $24,150 HoH standard deduction, no personal exemptions, 10%/12% brackets and a $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rules ($16,500 deduction, $10,100 of personal exemptions, a 15% bracket, $1,000 CTC) instead of OBBBA's permanent 2026 parameters. It also subtracted the $19,648 of employer-sponsored premiums from wages, even though gross income uses the full $62,000 of wages." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted all $12,824 of Social Security instead of the $10,900.40 taxable portion, used a $15,300 standard deduction instead of the $24,150 HoH amount, and used a $2,000 CTC instead of $2,200. Its $1,782 does not even follow from its own inputs, which give a tentative tax well above $5,000." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA individual provisions expire in 2026, so it restored personal exemptions, the 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent, giving a $24,150 HoH deduction, no exemptions, 10%/12% brackets and a $2,200 CTC. Its AGI of about $72,560 was essentially correct." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-expiration rules ($15,000 HoH standard deduction, $10,100 of personal exemptions, $1,000 CTC) instead of OBBBA's permanent 2026 parameters. It also subtracted the $19,648 of employer-sponsored premiums from the $62,000 of wages, which should be used in full." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path adds $10,900.40 of taxable Social Security to $62,000 of wages, subtracts $337.50 of educator expenses and the $24,150 HoH deduction, taxes $48,412.90 at 10%/12% ($5,455.55), and subtracts the $2,200 CTC. Its $3,043 is $212.55 too low, which equals about $1,770 of understated taxable income at 12%." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $2,000 CTC instead of the 2026 OBBBA amount of $2,200. Its implied $5,678 tentative tax is $222 above the correct $5,455.55, which matches using the pre-OBBBA $22,500 HoH standard deduction and a $300 educator cap instead of $24,150 and $337.50." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It applied personal exemptions, which are zero for 2026 because OBBBA permanently repealed them, so its taxable income and tax came out far too low. The correct taxable income is $48,412.90, including $10,900.40 of taxable Social Security after the $24,150 HoH deduction, giving $5,455.55 of tax less the $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It got the 85% taxable Social Security right but used the pre-OBBBA $22,500 HoH standard deduction instead of 2026's $24,150. It also ended the 10% bracket at $17,000 instead of $17,700, capped educator expenses at $300 instead of $337.50, and subtracted a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as the child's non-taxable income, so the $10,900.40 taxable portion never reached the head's AGI. It also used a $22,500 HoH standard deduction instead of $24,150 and a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Its AGI and $2,200 CTC were right, but it used a $24,350 standard deduction instead of $24,150. It also miscomputed the bracket tax: 10%/12% on about $48,212 is about $5,431, not $6,347, so the tax was overstated by roughly $900." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and credits wipe out the tax entirely. Taxable income is $48,412.90 after the $24,150 HoH deduction, which produces $5,455.55 of tax, far more than the $2,200 CTC, leaving $3,255.55." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It brought in itemized medical deductions and applied no credits beyond basic limits. The $3,060 of non-employer medical costs is far below the 7.5%-of-AGI floor, so the $24,150 HoH standard deduction applies, and the $2,200 CTC must be subtracted. It also never included the $10,900.40 of taxable Social Security." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It left out the $10,900.40 of taxable Social Security and used a $22,500 standard deduction instead of $24,150. It then set aside a $384 'refundable portion' and subtracted only $384.40 of nonrefundable CTC. Because the tentative tax exceeds the credit, the full $2,200 CTC is applied nonrefundably first." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It taxed only $6,412 (50%) of the Social Security benefits. Provisional income of about $68,075 is far above the $34,000 upper base for a head of household, so the 85% cap applies and $10,900.40 is taxable." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the right structure (HoH, taxable Social Security, $338 educator deduction, $24,150 standard deduction, $2,200 CTC) but showed no figures. Its $3,269 is $13.45 too high, which equals about $112 of extra taxable income at 12% compared with the exact $48,412.90." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left out the Social Security dependent benefits entirely. They belong on the head's return, with 85% ($10,900.40) taxable because provisional income is far above $34,000. Its brackets and $2,200 CTC were right, but its taxable income was $37,550 instead of $48,412.90." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It deducted 'limited mortgage interest' even though only a mortgage balance is listed, so the interest is $0 and the standard deduction applies. Its income also leaves out the $10,900.40 of taxable Social Security. Its $4,523 does not match $5,455.55 of tax on $48,412.90 less the full $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-expiration rules ($12,389 standard deduction, $10,733 of personal exemptions, a 15% bracket, $1,000 CTC) instead of OBBBA's permanent 2026 parameters: a $24,150 HoH deduction, no exemptions, 10%/12% brackets and a $2,200 CTC. Its $72,562 AGI was correct." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied 2026 'sunset' rules ($12,400 HoH standard deduction, two $5,350 personal exemptions, a 15% bracket, $1,000 CTC) instead of OBBBA's permanent $24,150 HoH deduction, no exemptions, 10%/12% brackets and $2,200 CTC. It also capped educator expenses at $300 instead of $337.50." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It miscomputed 85% of $12,824 as $10,890 instead of $10,900.40 and capped educator expenses at $300 instead of deducting $337.50. It then overstated the tax on its $48,440 of taxable income ($5,464 instead of $5,458.80), leaving the after-credit result about $8 too high." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA rules: a $12,538 standard deduction, $10,862 of personal exemptions, a 15% bracket from $17,902, and a $1,000 CTC. Under OBBBA, 2026 has a $24,150 HoH deduction, no exemptions, a 12% second bracket and a $2,200 CTC. Its $72,562 AGI was correct." +us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It capped educator expenses at $300 while the reference deducts the full $337.50, which adds $4.50 of tax. Separately, it computed taxable Social Security as $10,890 instead of $10,900.40 and rounded the tentative tax to about $5,460, so its total is about $3,260 instead of $3,255.55." +us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto loan interest, which is not deducted here, cutting taxable income to $48,137.40 instead of $48,412.90 and the tax by about $33. Everything else matches: 85% taxable Social Security, the $338 educator deduction, the $24,150 deduction, the 2026 brackets and the $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It filed single with the 2025 $15,000 deduction and single brackets. A parent with a 12-year-old qualifies for head of household, with a $24,150 deduction and a 10% bracket up to $17,700. It also left out the $10,900.40 of taxable Social Security and used a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It overstated taxable income by about $862 ($49,275 instead of $48,412.90) and ended the 10% bracket at $17,000 instead of $17,700. That gives a $5,573 tentative tax instead of $5,455.55; its $2,200 CTC was right." +us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed single ($16,100 deduction, single brackets reaching 22%) instead of head of household with the $24,150 deduction, and it deducted $275 of auto loan interest. It then let only $500 of the CTC reduce tax. Because tax exceeds the credit, the full $2,200 applies nonrefundably; the $1,700 refundable cap only limits the additional CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed married filing jointly, with MFJ Social Security base amounts and a $31,200 deduction. An unmarried parent with a 12-year-old files head of household, with a $24,150 deduction and HoH brackets. It also used a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the $32,200 MFJ deduction, excluded the dependent benefits, and claimed taxable income falls to $0, although $62,000 of wages minus even $32,200 leaves positive taxable income. The correct HoH computation gives $48,412.90 of taxable income, $5,455.55 of tax, and $3,255.55 after the $2,200 CTC." us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model computed pre-credit tax of about $4,300 against a $2,200 CTC and then contradicted its own arithmetic by awarding $1,700, writing that the nonrefundable portion ""absorbs tax"" while still paying out the refundable cap. It applied only two of the three ACTC limitations — the $1,700 per-child cap and the 15%-of-earnings-over-$2,500 phase-in — and omitted the controlling leg of IRC 24(d)(1)(B), the amount of CTC blocked by the tax-liability limitation, which is $0 when liability ($5,455.55 here) exceeds the entire $2,200 credit." us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model stated that ""tax liability is largely offset"" by the CTC and then treated the $1,700 refundable ceiling as the amount payable, inverting Schedule 8812: line 16a (total CTC minus the nonrefundable amount actually used) is $0 here, and the $1,700 cap can only limit that residual, never create one. It also used a stale $2,000 CTC rather than the 2026 OBBBA amount of $2,200, but the credit is fully absorbed at either figure since pre-credit tax is $5,455.55." us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model derived the correct answer explicitly — ""the full $2,000 CTC would be absorbed by tax liability, leaving no refundable Additional Child Tax Credit"" — then overrode it with $1,700 by invoking unnamed ""itemized/above-the-line deductions reducing taxable income further."" Reaching any ACTC requires taxable income below roughly $21,280 under the 10%/12% HoH schedule, i.e. about $51,300 of deductions against $72,562.90 of AGI; the actual itemizable items here (mortgage interest on a $55,000 balance, Georgia income tax, and medical costs entirely below the 7.5%-of-AGI floor) fall far short of even the $24,150 standard deduction." @@ -7296,52 +7388,53 @@ us,scenario_086,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"T us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component amounts—$3,844 of Social Security tax and $899 of Medicare tax—but failed to add them correctly. Those components total $4,743, not $3,979, and no Georgia-specific adjustment changes that sum." us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_086,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct $3,844 Social Security tax and $899 Medicare tax but submitted $3,813 instead of adding them. The stated components total $4,743." -us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used a $24,000 head-of-household standard deduction (the correct figure is $15,000), a $4,000 dependent exemption (correct: $5,000), and a 5.19% rate (correct: 4.99%). That produced its own figure of $1,749. It then submitted $2,732 under a 'single-filer' label, which contradicts its own calculation and matches no Georgia computation." -us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Subtracted $275 of auto loan interest, which is not a Georgia subtraction, so its Georgia AGI was $61,387 instead of $61,662.50. It also used a $24,000 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%. The overstated deduction drove taxable income down to $33,387 against the correct $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied Georgia's repealed six-bracket graduated schedule (1%–6%) to $62,000 of gross wages. It took no standard deduction or dependent exemption. Georgia's 2026 tax is a flat 4.99% on $41,662.50, which is Georgia AGI of $61,662.50 minus the $15,000 head-of-household deduction and the $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Used a 5.19% rate instead of Georgia's 2026 rate of 4.99%. Its $2,900 answer implies about $55,900 of taxable income, so it understated the $15,000 head-of-household standard deduction and $5,000 dependent exemption that bring taxable income to $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Used an $18,500 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave $2,032.51. It then submitted $2,191 with no supporting arithmetic." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%, which gave $2,324. It then submitted $1,869 with no supporting arithmetic." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Correctly excluded Social Security, but subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $4,000 dependent exemption instead of $5,000. It then taxed $45,700 at 5.09% instead of $41,662.50 at Georgia's 2026 rate of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Started from a federal AGI that already left out Social Security ($61,662), then subtracted the full $12,824 of benefits again. Georgia subtracts only the $10,900.40 of taxable Social Security that is in federal AGI. It also applied pre-2024 law: a $5,400 standard deduction, a $2,700 personal exemption, and a $3,000 dependent exemption at 5.39%, instead of $15,000 + $5,000 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave about $2,372. It then submitted an unexplained, inflated $2,765." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia income, but subtracted only a $12,000 standard deduction (correct: $15,000 for head of household) and a $4,000 dependent exemption (correct: $5,000). It then applied 5.09% instead of 4.99%, taxing $45,662 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Never subtracted the $10,900.40 of taxable Social Security, so it started from $72,600 instead of Georgia AGI of $61,662.50. It also used a $9,000 standard deduction and two $2,700 personal exemptions instead of $15,000 + $5,000, at 5.29% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied pre-2024 Georgia law: a $5,400 standard deduction and $2,700 exemptions for the head and the child. Current law allows a $15,000 head-of-household standard deduction and a $5,000 dependent exemption with no personal exemption. It also used 5.29% instead of 4.99%, giving taxable income of $49,340 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Used old-law deductions totaling $11,400: a $5,700 standard deduction plus $2,700 and $3,000 exemptions. The 2026 amounts total $20,000: a $15,000 head-of-household standard deduction plus a $5,000 dependent exemption. As a result it taxed $50,300 instead of $41,662.50, and at 5.09% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Used the correct 4.99% rate but started from wages without the $338 educator expense adjustment. It subtracted a $12,000 standard deduction instead of $15,000, and $6,700 of exemptions (a nonexistent personal exemption plus a dependent amount) instead of the single $5,000 dependent exemption. Taxable income came out at $43,300 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $19,648 employer-sponsored insurance premiums from the stated wages. It also never removed the $10,900.40 of taxable Social Security from Georgia income. It then used a $15,800 'personal exemption', a $3,000 dependent exemption, and a 5.39% rate, instead of the $15,000 standard deduction, $5,000 dependent exemption, and 4.99% rate." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its $2,690 answer implies roughly $52,000–$54,000 of Georgia taxable income, which is consistent with taxing the $62,000 of wages after only about $8,000–$10,000 of deductions. The correct deductions are $20,000 (the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption) against $61,662.50 of Georgia AGI, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia AGI, but used a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It then taxed $46,662 at 5.39% instead of $41,662.50 at Georgia's 2026 rate of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Started from a $52,952 'federal AGI' built by subtracting the $19,648 employer-sponsored premiums from wages, and did not take out the taxable Social Security. It then subtracted only $15,000 of total deductions and exemptions instead of $20,000, and applied 5.29% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $3,212 answer implies about $60,000–$64,000 of taxable income, which exceeds Georgia AGI after deductions. That is consistent with leaving the taxable Social Security in Georgia income and applying little or no deduction, instead of taxing $41,662.50 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $3,047 answer implies roughly $56,500–$61,000 of taxable income. That is consistent with skipping most of the $15,000 head-of-household standard deduction and $5,000 dependent exemption, or with leaving taxable Social Security in Georgia income, instead of taxing $41,662.50 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $1,432.32 answer implies only about $26,500–$28,700 of taxable income, roughly $13,000–$15,000 below the correct $41,662.50. That is consistent with also subtracting the $19,648 employer-sponsored premiums or otherwise overstating deductions beyond the $15,000 standard deduction and $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Used a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It also took Georgia AGI as $61,700 by capping educator expenses at $300. It then taxed $46,700 at 5.39% instead of $41,662.50 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia AGI, but used a $10,600 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption for the 12-year-old. It then applied 5.39% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Used a $24,000 standard deduction (the married-joint level) instead of the $15,000 head-of-household amount. It also added about $8,400 of personal and dependent exemptions instead of the single $5,000 dependent exemption, deducting $32,400 instead of $20,000. That cut taxable income to $29,262, which it then taxed at 5.19% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Its $1,639 answer implies only about $31,600–$32,800 of taxable income. That means it deducted roughly $9,000 more than the $15,000 head-of-household standard deduction plus $5,000 dependent exemption, and so understated the correct $41,662.50 taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Its $498 answer implies only about $10,000 of Georgia taxable income, which reflects grossly overstated deductions or adjustments. The correct computation subtracts only $20,000 (the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption) from $61,662.50 of Georgia AGI, then taxes $41,662.50 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used a $12,000 standard deduction instead of the $15,000 head-of-household amount and $4,500 of exemptions instead of the $5,000 dependent exemption. It then applied 5.19% instead of 4.99%, taxing $45,162 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Correctly excluded taxable Social Security, but its $1,357.80 answer implies only about $25,000–$27,000 of taxable income. That means it deducted roughly $35,000 instead of the $20,000 total from the $15,000 head-of-household standard deduction and $5,000 dependent exemption." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Correctly reached $61,662.40 of Georgia income, but subtracted a $12,000 exemption instead of the $15,000 head-of-household standard deduction and a $4,000 dependent exemption instead of $5,000. It then applied a 5.09% rate instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $1,903 at 5.19% implies about $36,700 of taxable income. That means it deducted about $25,000 from $61,662 of Georgia AGI instead of $20,000 (the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption), and it used 5.19% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Used a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $4,000 dependent exemption instead of $5,000. It then taxed $45,662 at a 5.09% rate instead of $41,662.50 at Georgia's 2026 rate of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Applied the correct 4.99% rate, but used a $12,000 standard deduction plus a $4,000 'taxpayer exemption' that Georgia no longer allows, and a $3,000 dependent exemption. Current law gives a $15,000 head-of-household standard deduction plus a $5,000 dependent exemption. It also capped educator expenses at $300, leaving taxable income at $42,700 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Used a $12,000 head-of-household deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000. It then applied 5.19% instead of 4.99%, taxing $45,662 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Correctly reached $61,662 of Georgia AGI, but used a $12,000 head-of-household deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000. It then applied 5.09% instead of Georgia's 2026 rate of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Its $2,184 answer implies about $40,500–$43,800 of taxable income at the 5.0–5.4% rates in circulation. At 4.99% it implies about $43,800, about $2,100 above the correct $41,662.50, which is consistent with understating the $15,000 head-of-household standard deduction and $5,000 dependent exemption or using an outdated higher rate." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Used an $18,000 head-of-household standard deduction instead of $15,000 and entirely omitted the $5,000 dependent exemption for the child. It then applied a 5.29% rate instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used a $12,000 head-of-household standard deduction instead of $15,000, plus $5,700 of personal ($2,700) and dependent ($3,000) exemptions instead of the single $5,000 dependent exemption. It then applied 5.39% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Correctly subtracted taxable Social Security, but used a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It then taxed $46,700 at 5.19% instead of $41,662.50 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Subtracted the $338 educator expense a second time, even though federal AGI already reflects it. It also applied pre-2024 law: a $5,400 standard deduction and two $2,700 personal exemptions, instead of the $15,000 head-of-household deduction and $5,000 dependent exemption. Finally, it used the 2024 rate of 5.49% instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Reached roughly the right taxable base (about $40,700 against $41,662.50) after excluding Social Security. However, it applied a 5.29% flat rate instead of Georgia's 2026 rate of 4.99%, which overstated tax by roughly $70." -us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Applied the correct 4.99% rate to the correct $61,662 of Georgia AGI. However, it used a $12,000 standard deduction instead of the $15,000 head-of-household amount, plus two $2,700 personal exemptions instead of a single $5,000 dependent exemption. Taxable income came out at $44,262 instead of $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Removed an extra $275 of auto loan interest, which put Georgia AGI at $61,387 instead of $61,662.50. It also used an $18,500 head-of-household standard deduction instead of $15,000, a $3,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Applied pre-2024 law: a $5,000 standard deduction and a $2,700 personal exemption, with no exemption for the dependent child. Current law gives a $15,000 head-of-household deduction plus a $5,000 dependent exemption. It then taxed $54,000 at 5.19% instead of $41,662.50 at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its taxable income of $41,687 nearly matches the correct $41,662.50, but only because offsetting errors cancelled out. It subtracted the auto loan interest, used a $12,000 standard deduction, added a nonexistent $3,700 personal exemption, and used a $4,000 dependent exemption, instead of the $15,000 standard deduction plus the $5,000 dependent exemption. The remaining error is that it applied 5.09% instead of Georgia's 2026 rate of 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Included the full $12,824 of Social Security dependent benefits in Georgia income, even though Georgia fully subtracts the taxable Social Security in federal AGI. It also used a $12,000 standard deduction, a $2,700 personal exemption, and a $3,000 dependent exemption at 5.12%, instead of a $15,000 head-of-household deduction and a $5,000 dependent exemption at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Subtracted its deduction from $74,824 of gross income that still included the Social Security benefits, despite saying they were excluded. It also used an invented $23,100 standard deduction, producing $51,700 of taxable income instead of $41,662.50. Its $2,361.45 implies a 4.57% rate that matches no Georgia rate." +us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the $24,000 married-filing-jointly figure as the head-of-household standard deduction, a $4,000 dependent exemption, and a 5.19% rate instead of $15,000, $5,000 and 4.99%. It then submitted $2,732, which does not match its own $1,749 calculation. The correct result is $41,662.50 × 4.99% = $2,078.96." +us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It wrongly subtracted the $275 of auto loan interest from Georgia AGI; personal interest is not a Georgia subtraction. It also used the $24,000 joint figure as the head-of-household standard deduction, a $4,000 dependent exemption, and a 5.09% rate. That gave $33,387 of taxable income instead of $41,662.50 taxed at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied Georgia's old graduated brackets (1%–6%, single filer), which HB 1437 replaced with a flat tax, rather than the 2026 flat rate of 4.99%. It also skipped the Social Security subtraction, the $15,000 head-of-household standard deduction and the $5,000 dependent exemption." +us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It used a 5.19% rate instead of 4.99%. At 5.19%, $2,900 implies about $55,900 of taxable income. The model therefore applied only about $6,000 of deductions, not the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption that leave $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an $18,500 head-of-household deduction, a $4,000 dependent exemption and a 5.19% rate instead of $15,000, $5,000 and 4.99%. It then replaced its own $2,032.51 result with $2,191, an amount its reasoning does not support." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It treated the head as a single filer with a $12,000 standard deduction instead of head of household with $15,000. It also used a $4,000 dependent exemption instead of $5,000 and a 5.09% rate instead of 4.99%. It then arbitrarily changed its own $2,324 result to $1,869." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction and a $4,000 dependent exemption instead of the $15,000 head-of-household deduction and $5,000 exemption. It also used a 5.09% rate instead of 4.99%, and $300 of educator expenses instead of $338. Correct taxable income is $41,662.50, not $45,700." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Its federal AGI already excluded Social Security, yet it subtracted the full $12,824 of benefits again. Georgia subtracts only the $10,900.40 that is federally taxable, from an AGI that includes it. It also filed as single and used the pre-HB 1437 amounts ($5,400 standard deduction, $2,700 personal exemption, $3,000 dependent exemption) with a 5.39% rate instead of $15,000 + $5,000 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used the $12,000 single standard deduction and a $4,000 dependent exemption at 5.19%, which gives $2,372. It then inflated that to $2,765 with no basis. The correct calculation uses $15,000 + $5,000 of deductions and 4.99%, giving $2,078.96." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The Georgia AGI of $61,662 is right. However, it used a $12,000 standard deduction and a $4,000 dependent exemption instead of $15,000 for head of household and $5,000 per dependent, and a 5.09% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It taxed the full federal AGI of about $72,600 without subtracting the $10,900.40 of taxable Social Security that Georgia excludes. It also used a $9,000 standard deduction with two $2,700 personal exemptions and a 5.29% rate, instead of $15,000 + $5,000 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It started from an unexplained $60,140 AGI instead of $61,662.50. It also applied pre-HB 1437 amounts ($5,400 standard deduction plus $2,700 exemptions for the head and the child) at 5.29%, instead of the $15,000 head-of-household deduction and $5,000 dependent exemption at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used outdated amounts ($5,700 standard deduction, a $2,700 filer exemption and a $3,000 child exemption), leaving $50,300 of taxable income. The correct deductions are $15,000 + $5,000, leaving $41,662.50. It also used 5.09% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The 4.99% rate is right. However, it started from the $62,000 of gross wages instead of a Georgia AGI of $61,662.50 (net of the $338 educator expense). It also used a $12,000 deduction plus $6,700 of exemptions instead of the $15,000 head-of-household deduction and $5,000 dependent exemption, leaving $43,300 taxable instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $19,648 of employer-sponsored insurance premiums from AGI, but these are employer-paid and not a deduction, which gave a $52,952.40 AGI. It never subtracted the $10,900.40 of taxable Social Security. It also used a $15,800 exemption, a $3,000 dependent exemption and a 5.39% rate instead of $15,000 + $5,000 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation. The correct calculation is $61,662.50 − $15,000 − $5,000 = $41,662.50, taxed at 4.99%. Its $2,690 corresponds to about $50,000–$54,000 of taxable income, which means it applied roughly $12,000 of deductions to wages, left out the $5,000 dependent exemption, and used an outdated rate." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The Georgia AGI of $61,662 is right. However, it used a $12,000 head-of-household standard deduction and a $3,000 dependent deduction instead of $15,000 and $5,000, and a 5.39% rate instead of the 2026 rate of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a $52,952 federal AGI, which comes from wrongly subtracting the $19,648 of employer insurance premiums and not subtracting the $10,900.40 of taxable Social Security. Georgia AGI is $61,662.50. It also deducted only $15,000 in total instead of $15,000 + $5,000, and taxed the result at 5.29% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,212 corresponds to about $60,000 of taxable income at a rate above 5%. That means it taxed roughly the full Georgia AGI without the $15,000 head-of-household deduction and $5,000 dependent exemption, and did not use the 4.99% rate on $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,047 matches federal AGI ($72,562) minus about $16,000 of deductions, taxed at 5.39%. That means it did not subtract the $10,900.40 of taxable Social Security that Georgia excludes, and it used the wrong deduction amounts and rate instead of $15,000 + $5,000 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $1,432.32 corresponds to only about $27,000–$29,000 of taxable income, far below $41,662.50. That means it took far more than the correct $20,000 of deductions, for example a $24,000 joint-sized standard deduction or the employer premiums subtracted from AGI." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $12,000 head-of-household standard deduction and a $3,000 dependent exemption instead of $15,000 and $5,000, and a 5.39% rate instead of 4.99%. It also used $300 of educator expenses rather than the $338 PolicyEngine allows, which put Georgia AGI at $61,700." +us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The Georgia AGI of $61,662 is right. However, it used a $10,600 head-of-household standard deduction and skipped the $5,000 dependent exemption entirely, and it taxed at 5.39% instead of $15,000 + $5,000 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the $24,000 joint figure as the head-of-household standard deduction and added about $8,400 of personal and dependent exemptions, deducting $32,400 instead of $20,000. It also used 5.19% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $1,639 corresponds to about $32,000 of taxable income at a 5% rate. That means it deducted roughly $10,000 too much, missing the correct $15,000 head-of-household deduction plus $5,000 dependent exemption, which leave $41,662.50 taxed at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It built state taxable income from a federal-style base with medical and other adjustments. Its $498 implies only about $10,000 of taxable income instead of $41,662.50. Georgia takes only its own $15,000 head-of-household deduction and $5,000 dependent exemption from the $61,662.50 Georgia AGI." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $12,000 head-of-household standard deduction and $4,500 of exemptions instead of $15,000 and $5,000, and a 5.19% rate instead of 4.99%. That gave $45,162 of taxable income instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It correctly removed taxable Social Security. However, $1,357.80 implies only about $26,000–$27,000 of taxable income, meaning it deducted roughly $35,000 instead of the $15,000 head-of-household deduction plus $5,000 dependent exemption. The correct tax is $41,662.50 × 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used $12,000 and $4,000 of deductions instead of the $15,000 head-of-household standard deduction and $5,000 dependent exemption, and a 5.09% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a 5.19% rate instead of 4.99%. At that rate, $1,903 implies about $36,700 of taxable income, so it deducted roughly $25,000 from Georgia AGI instead of $15,000 + $5,000 = $20,000." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction and a $4,000 dependent exemption instead of the $15,000 head-of-household deduction and $5,000 dependent exemption. It also used a 5.09% rate instead of the 2026 rate of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The 4.99% rate is right. However, it invented a $4,000 taxpayer exemption, even though Georgia no longer has a personal exemption, and used a $12,000 standard deduction and $3,000 dependent exemption instead of $15,000 and $5,000. It also used $300 of educator expenses instead of $338, leaving $42,700 taxable instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It used a $12,000 head-of-household deduction and a $4,000 dependent exemption instead of $15,000 and $5,000, and taxed $45,662 at 5.19% instead of $41,662.50 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The Georgia AGI of $61,662 is right. However, it used a $12,000 head-of-household deduction and a $4,000 dependent exemption instead of $15,000 and $5,000, and a 5.09% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no figures. Its $2,184 corresponds to about $42,000–$43,800 of taxable income at a rate between 4.99% and 5.19%. That means it used deduction amounts and a rate that differ from the correct $15,000 head-of-household deduction, $5,000 dependent exemption and 4.99% applied to $61,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used an $18,000 head-of-household standard deduction and left out the $5,000 dependent exemption, leaving $43,662 of taxable income instead of $41,662.50. It also used 5.29% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The Social Security subtraction is right. However, it used a $12,000 head-of-household deduction and $5,700 of personal and dependent exemptions instead of $15,000 + $5,000, and a 5.39% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $12,000 head-of-household standard deduction and a $3,000 dependent exemption instead of $15,000 and $5,000, which left $46,700 taxable instead of $41,662.50. It also used a 5.19% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $338 educator expense a second time, even though federal AGI already reflects it. It also used pre-HB 1437 amounts ($5,400 standard deduction plus two $2,700 exemptions) and a 5.49% rate instead of the $15,000 head-of-household deduction, $5,000 dependent exemption and 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its taxable income of about $40,700 is close to the correct $41,662.50. The main error is the rate: it used 5.29% instead of Georgia's 2026 flat rate of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The 4.99% rate and $61,662 Georgia AGI are right. However, it applied a $12,000 standard deduction and two $2,700 personal exemptions, which Georgia no longer allows. The correct amounts are the $15,000 head-of-household deduction and a $5,000 dependent exemption, which leave $41,662.50 taxable instead of $44,262." +us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It wrongly subtracted the $275 of auto loan interest, which gave a $61,387 AGI. It also used an $18,500 head-of-household deduction and a $3,000 dependent exemption instead of $15,000 and $5,000, and a 5.19% rate instead of 4.99%. Its result lands near the reference only because these errors offset each other." +us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It started from gross wages and applied outdated amounts: a $5,000 standard deduction and a $2,700 personal exemption, with no dependent exemption. It also used a 5.19% rate. The correct calculation takes the $15,000 head-of-household deduction and $5,000 dependent exemption from $61,662.50 and applies 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Its taxable income of about $41,000 is close to the correct $41,662.50. The error is the rate: it used 5.19% instead of the 2026 flat rate of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It wrongly subtracted the $275 of auto loan interest from federal AGI. It also used a $12,000 single deduction, a $3,700 personal exemption and a $4,000 dependent exemption instead of the $15,000 head-of-household deduction and $5,000 dependent exemption. The decisive error is the 5.09% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It included the full $12,824 of Social Security in Georgia taxable income instead of starting from the $61,662.50 Georgia AGI, which already excludes taxable Social Security. It also used $12,000 + $2,700 + $3,000 of deductions and a 5.12% rate instead of $15,000 + $5,000 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from gross income of $74,824, including Social Security, instead of a Georgia AGI of $61,662.50. It then subtracted a $23,100 federal-style standard deduction instead of Georgia's $15,000 head-of-household deduction and $5,000 dependent exemption, and applied an effective rate of about 4.57% instead of 4.99%." us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Georgia refundable dependent-child credit of $260 per qualifying child. Georgia has no such applicable refundable credit, so the 12-year-old dependent generates $0 in state refundable credits." us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the deduction for the $9,600 of qualified tips included in wages, leaving $7,085 of purported taxable income. It also incorrectly included the $675 state tax refund in gross income without establishing a prior-year itemized-deduction tax benefit." us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the required output contract." @@ -7521,26 +7614,27 @@ us,scenario_090,snap,claude-sonnet-5.5,llm_error,asset_resource,False,"The model us,scenario_090,snap,glm-5.3,llm_error,asset_resource,False,"The model applied only the 130% FPG gross-income test and the benefit formula and never tested the head's $20,300 bank balance against the SNAP resource limit. Kansas has no BBCE, so the $3,000 federal resource limit makes the household ineligible, giving $0 rather than $1,286." us,scenario_090,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model asserted that the household has 'low net resources' even though the head holds $20,300 in bank assets. Kansas does not use BBCE, so that balance exceeds the federal $3,000 SNAP resource limit and makes the household ineligible. The correct value is $0, not the model's unsupported $438." us,scenario_090,snap,grok-build-0.1,llm_error,asset_resource,False,"The model stated 'Kansas uses BBCE so no asset test' and skipped the resource test. Kansas does not use broad-based categorical eligibility, so the $20,300 bank balance exceeds the federal $3,000 resource limit, the household is ineligible, and SNAP is $0, not $1,112." -us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted a $3,500 standard deduction and a $2,250 personal exemption, then taxed $1,270 at 5.2% to get $66. The 2024 Kansas law's $9,160 single personal exemption plus the $3,605 standard deduction totals $12,765, which is more than the $7,020 KAGI, so Kansas taxable income and tax are both $0." -us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $3,605 standard deduction but the pre-2024 $2,250 personal exemption, which left $1,165 taxed at 5.2%. Kansas's current single personal exemption is $9,160, so the $12,765 in combined deductions wipes out the $7,020 KAGI and the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a standard deduction of about $5,100 ('one-third of federal'), left out the personal exemption entirely, and taxed $1,920 at 5.2%. The actual Kansas single standard deduction is $3,605 and the personal exemption is $9,160, which together exceed the $7,020 KAGI and give $0 tax." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted only a $3,500 standard deduction, skipped the personal exemption, and taxed $3,520 at the repealed 3.1% bottom rate. Kansas's $9,160 single personal exemption plus the $3,605 standard deduction exceed the $7,020 KAGI, so taxable income is $0." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $3,500 standard deduction and the old $2,250 personal exemption, then taxed the remaining $1,270 at 5.2%. Under the 2024 Kansas law, the single personal exemption is $9,160 and the standard deduction is $3,605, so the $7,020 KAGI is fully sheltered and the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $3,800 standard deduction and a $2,320 exemption, which is the Kansas dependent amount rather than the $9,160 single-filer exemption. It then taxed $900 at the repealed 3.1% rate. The correct $3,605 + $9,160 = $12,765 exceeds the $7,020 KAGI, so the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It paired a $3,500 standard deduction with a $2,320 exemption, which is the dependent amount, and taxed $1,200 at 5.2%. It never applied the $9,160 single personal exemption, which together with the $3,605 standard deduction brings Kansas taxable income to $0 with no need for any low-income exclusion." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $3,600 standard deduction and a $2,320 personal exemption, leaving $1,100 taxed at 5.2%. The single-filer exemption under current Kansas law is $9,160, not the $2,320 dependent figure, so the combined $12,765 exceeds the $7,020 KAGI and the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used the $3,605 standard deduction but the obsolete $2,250 exemption, then taxed $1,165 at the repealed 3.1% rate. Kansas's $9,160 single personal exemption alone exceeds the income left after the standard deduction, so the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It applied the pre-2024 Kansas amounts ($3,500 standard deduction, $2,250 exemption, 3.1% bottom rate) to get $1,270 × 3.1%. Under the 2024 law, the $3,605 standard deduction plus the $9,160 single exemption exceed the $7,020 KAGI, so the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the old $3,500 standard deduction and $2,250 exemption and taxed $1,270 at the repealed 3.1% rate. It missed the 2024 increase of the single personal exemption to $9,160, which together with the $3,605 standard deduction leaves $0 Kansas taxable income." -us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted only a $3,500 standard deduction, left out the personal exemption entirely, and taxed $3,520 at the repealed 3.1% rate. Kansas's $9,160 single personal exemption plus the $3,605 standard deduction exceed the $7,020 KAGI, so the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted only a standard deduction of about $3,700, never applied a personal exemption, and used a nonexistent 5.15% flat rate on $3,320. Kansas allows a $9,160 single personal exemption on top of the $3,605 standard deduction, which takes taxable income to $0." -us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,120 of taxable income means it subtracted only about $5,900 in combined standard deduction and exemption, which fits the old ~$2,300 per-person exemption. The current Kansas single exemption is $9,160, so the $12,765 in combined deductions leaves $0 to tax at 5.2%." -us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,605 standard deduction but only a $2,250 personal exemption, then taxed $1,165 at 5.2% and rounded to $61. The 2024 Kansas law sets the single personal exemption at $9,160, which zeroes out Kansas taxable income on $7,020 of KAGI." -us,scenario_090,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its $1,165 of taxable income means it subtracted about $5,855, which matches the $3,605 standard deduction plus the obsolete $2,250 exemption. It then taxed that at 5.2%. The correct $9,160 single personal exemption plus the $3,605 standard deduction exceed the $7,020 KAGI, so the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It paired the correct $3,605 standard deduction with a $2,320 exemption, which is the Kansas dependent amount, and taxed $1,095 at 5.2%. A single filer's own exemption is $9,160, so combined deductions of $12,765 exceed the $7,020 KAGI and the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted only a $3,500 standard deduction, skipped the personal exemption, and applied a made-up 5.7% flat rate to $3,520. Kansas's $3,605 standard deduction plus its $9,160 single personal exemption exceed the $7,020 KAGI, so the tax is $0." -us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted a standard deduction of about $3,600 and an exemption of about $2,300, which is the old per-person figure, then taxed about $1,100 at an assumed 5.5% flat rate. The current Kansas single exemption of $9,160 plus the $3,605 standard deduction leave $0 of taxable income." -us,scenario_090,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It treated the personal exemption as a made-up $110 credit rather than a deduction from income, so it taxed $3,520 at 5.2% and subtracted $110. Kansas deducts a $9,160 single personal exemption along with the $3,605 standard deduction, which exceeds the $7,020 KAGI and gives $0 tax." +us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the pre-2024 $2,250 personal exemption and a $3,500 standard deduction, leaving $1,270 taxed at 5.2%. It missed that SB 1 (2024) raised the single-filer exemption to $9,160, which together with the standard deduction exceeds the $7,020 AGI and brings Kansas taxable income to $0." +us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used the correct $3,605 standard deduction but paired it with the repealed $2,250 personal exemption instead of the $9,160 single-filer exemption from SB 1. Total deductions of about $12,765 exceed the $7,020 AGI, so tax is $0, not $60.58." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model made up a standard deduction of 'one-third of federal' (about $5,100) and left out the Kansas personal exemption entirely. The actual $3,605 standard deduction plus the $9,160 single exemption wipes out the $7,020 AGI, leaving $0 tax." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $3,500 standard deduction, left out the $9,160 single-filer personal exemption, and applied the repealed 3.1% bottom rate. With the exemption included, Kansas taxable income is $0." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model used the old $2,250 personal exemption and a $3,500 standard deduction, leaving $1,270 taxed at 5.2%. Since SB 1 the single exemption is $9,160, which together with the standard deduction exceeds the $7,020 AGI, so tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model used a $2,320 exemption, which is the per-dependent amount, instead of the $9,160 single-filer personal exemption. It also applied the repealed 3.1% rate. The correct deductions exceed the $7,020 AGI, so taxable income and tax are $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted $3,500 plus a $2,320 exemption (the dependent amount) instead of the $9,160 single-filer personal exemption. It also cited a $7,000 'low-income exclusion' threshold that is not part of current Kansas law. The correct deductions alone bring Kansas taxable income to $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a $2,320 personal exemption instead of the $9,160 single-filer exemption enacted in 2024. As a result it found $1,100 of taxable income where the correct deductions leave $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied the old $2,250 personal exemption rather than the $9,160 single exemption, and it taxed at the repealed 3.1% rate. The $3,605 standard deduction plus the $9,160 exemption exceeds the $7,020 AGI, so tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model used the pre-2024 $3,500 standard deduction, the $2,250 exemption, and the 3.1% bracket. It missed SB 1's $9,160 single-filer personal exemption, which brings Kansas taxable income to $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model applied the old $2,250 exemption and the 3.1% rate rather than the current $9,160 single-filer exemption. Deductions exceed the $7,020 AGI, so there is no Kansas taxable income." +us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $3,500 standard deduction and left out the Kansas personal exemption entirely, which is $9,160 for a single filer. It also used the repealed 3.1% rate. Including the exemption reduces taxable income to $0." +us,scenario_090,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model left out the $9,160 single-filer personal exemption and applied a made-up 5.15% flat rate to $3,320. The standard deduction plus the exemption exceeds the $7,020 AGI, so tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,120 of taxable income implies about $5,900 of combined deductions, which matches the old roughly $2,250–$2,320 exemption. Under SB 1 the single-filer exemption is $9,160, which drives taxable income to $0." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used the correct $3,605 standard deduction but the superseded $2,250 personal exemption, leaving $1,165 taxed at 5.2%. The $9,160 single exemption wipes out the remaining AGI, so tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model's $1,165 of taxable income reflects the old $2,250 personal exemption rather than the $9,160 single-filer exemption from 2024. The correct exemption brings Kansas taxable income and tax to $0." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model used $2,320, which is the per-dependent exemption, as the filer's personal exemption instead of the $9,160 single-filer amount. That left $1,095 of taxable income where the correct figure is $0." +us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted only the standard deduction and left out the $9,160 single-filer personal exemption. It also applied a 5.7% flat rate that Kansas does not have, since the current rates are 5.2% and 5.58%. With the exemption, taxable income is $0." +us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an exemption of about $2,300 instead of the $9,160 single-filer exemption and estimated a rate of about 5.5%. Correct deductions exceed the $7,020 AGI, so Kansas tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model used a $2,380 personal exemption, an outdated figure, rather than the $9,160 single-filer exemption. That left $940 taxable where the correct deductions leave $0." +us,scenario_090,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model treated the personal exemption as a $110 tax credit. Kansas has no such credit: its personal exemption is a $9,160 deduction from income for a single filer. That deduction, together with the standard deduction, exceeds the $7,020 AGI and leaves $0 of tax." us,scenario_090,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It priced the Kansas food sales tax credit off an income test alone, inventing an ""under-60, AGI below the poverty guideline"" rule; the actual credit (K.S.A. 79-32,271) requires the filer to be 55 or older, blind or disabled, or to have a dependent child under 18 residing in the home, and a 22-year-old single filer with no dependents meets none of those categories. It also missed that the credit has been nonrefundable since tax year 2013, so it cannot enter state_refundable_credits at all. Its $60 corresponds to no Kansas parameter — the statutory amount is $125 per exemption." us,scenario_090,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"It applied the 7.65% childless phase-in rate to $7,000 of wages to get a $535.50 federal EITC while ignoring the childless-EITC age requirement of 25 through 64; the head is 22, so the federal EITC is $0 and the Kansas EITC — which is a flat percentage of the federal credit — is $0. It compounded this by using an 18% Kansas match; the Kansas rate is 17%, though at a $0 federal base either rate yields $0." us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It used the correct 17% Kansas match but multiplied it against a federal EITC of $535.50 that does not exist: the childless EITC is available only to filers aged 25 through 64, and the head is 22, so the federal credit phases in to $0 and 17% of $0 is $0. Kansas has no independent earned-income credit computation, so a zero federal EITC forces ks_eitc to zero." @@ -7620,53 +7714,54 @@ us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was nothing to score. The correct answer is 7.65% of $27,000 in wages, or $2,065.50." us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium on top of the $27,000 gross wages, taxing $27,628. The prompt states that gross wages already include overtime pay, so this counts the overtime twice. The correct base is $27,000, giving $2,065.50." us,scenario_091,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,170 of capital gain distributions at 70%, giving $35,832 instead of Wisconsin AGI of $35,012.56. It phased the standard deduction down to about $10,668 instead of $12,067.66 and never subtracted the $700 personal exemption. It also submitted $1,257 even though its own math came to about $975; the correct taxable income of $22,244.90 gives $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of non-Schedule-D gains that are left out of the $35,012.56 Wisconsin AGI. It also skipped the $700 personal exemption, so taxable income came to $24,184 instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal taxable income using the federal standard deduction, rather than Wisconsin AGI with Wisconsin's sliding-scale deduction and $700 exemption. It also left out the $8,982 of interest and dividends. It then cut the tax to $0 with the federal American Opportunity Credit, which has no Wisconsin version; Wisconsin taxable income of $22,244.90 gives $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It said taxable income was low enough for $0 tax, ignoring $4,806 of interest and $4,176 of dividends. Those bring Wisconsin AGI to $35,012.56 and taxable income to $22,244.90 after the $12,067.66 standard deduction and $700 exemption, which is $843.66 of tax." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 of gains in a $36,183 AGI and phased the standard deduction down to about $9,500 instead of $12,067.66, which inflated taxable income to $25,983. It then submitted $1,140, above even its own $1,014." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It cut the sliding-scale standard deduction to about $7,700 instead of $12,067.66 and kept the $1,170 of gains in a $36,183 AGI, giving $27,783 of taxable income instead of $22,244.90. It then added about $70 for unnamed 'adjustments'." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It cut the standard deduction to about $9,000 instead of $12,067.66 and started from $36,183, which includes the $1,170 of gains. That gave about $26,500 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It kept 70% of the $1,170 in Wisconsin income ($35,832) after a 30% capital gain exclusion. The engine leaves these non-Schedule-D distributions out entirely, giving $35,012.56. The extra $819, plus a slightly low $12,003 deduction, raised taxable income to $23,129 instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used 2024 parameters: a $12,230 deduction phased out at 12% above $15,000. That cut the deduction to $9,688 instead of the 2026 sliding-scale amount of $12,067.66. Combined with the $1,170 of gains in a $36,183 AGI, taxable income came to $25,795 instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 of gains in a $36,183 AGI and used an $11,340 deduction instead of $12,067.66, giving $24,143 of taxable income instead of $22,244.90. It then subtracted a 'Married/Single credit' that does not apply to this single filer." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It phased the standard deduction down to about $11.6k instead of $12,067.66 and kept the $1,170 of gains in income. That gave about $23.9k of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used an unphased $13,450 standard deduction instead of the $12,067.66 phased amount, and kept the $1,170 of gains in a $36,183 AGI. These two errors roughly cancelled, giving $22,033 of taxable income instead of $22,244.90. It also used a $14,320 first bracket, which is not the 2026 threshold." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It used the old 3.54%/4.65% rates instead of 2026's 3.5%/4.4%. It applied an unphased $13,230 deduction, skipped the $700 exemption, and started from an unexplained $34,383 AGI instead of $35,012.56." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains that are left out of the $35,012.56 Wisconsin AGI. It subtracted a lump $14,050 instead of the $12,067.66 phased deduction plus $700 exemption, and used a $15,200 first bracket, giving $22,133 of taxable income and $837.05." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It cut the standard deduction to $9,963.40 instead of $12,067.66 and never subtracted the $700 personal exemption. Starting from a $36,183 AGI that includes the $1,170 of gains, it got $26,219.60 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its $29,594 AGI equals $36,183 minus the $6,589 of employer-sponsored insurance premiums, which are not a wage reduction here. That drops Wisconsin income well below the correct $35,012.56, giving $17,484 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It treated Wisconsin AGI as $27,000 of wages alone. It left out the $926.16 401(k) deferral and the $4,806 of interest and $4,176 of dividends that bring Wisconsin AGI to $35,012.56. Its $905 does not follow from any stated deduction or bracket calculation." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"At 3.5%/4.4%, $588 implies about $16,400 of taxable income, roughly $5,800 below the correct $22,244.90. That matches subtracting the $6,589 of employer-sponsored insurance premiums from income, as other Gemini models did, instead of starting from the $35,012.56 Wisconsin AGI." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 of employer-sponsored insurance premiums to get a $29,594 AGI, instead of the correct $35,012.56. It also skipped the $700 personal exemption, giving $16,887 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"With no derivation given, $1,177 implies about $29,800 of taxable income, roughly $7,500 above the correct $22,244.90. That is consistent with phasing the standard deduction down far below the correct $12,067.66 from a $35,012.56 Wisconsin AGI." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 60% capital gain exclusion, which Wisconsin does not have; Wisconsin's exclusion is 30%, and the engine leaves these non-Schedule-D distributions out of income entirely. It also subtracted a 'medical expense credit' that Wisconsin does not allow on these facts, pushing the tax below the correct $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $944 while its explanation says $1,018. It treated the $700 exemption as a 'personal exemption credit'; Wisconsin allows it as a deduction from income. Both figures imply taxable income $2,300 to $4,000 above the correct $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"$1,113 implies about $28,400 of taxable income, roughly $6,100 above the correct $22,244.90. That is consistent with under-applying the sliding-scale standard deduction, which is still $12,067.66 at $35,012.56 of Wisconsin AGI." -us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It said the Wisconsin standard deduction is fully phased out to $0 for a single filer at this income. At $35,012.56 of Wisconsin AGI it is still $12,067.66. It also skipped the $700 exemption, included the $1,170 of gains, and reached the 5.3% bracket on income that should be taxed only at 3.5%/4.4%." -us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It said the standard deduction is fully phased out and taxed the whole $36,183. The sliding-scale deduction is still $12,067.66 at this income, and the $700 exemption also applies, giving $22,244.90 of taxable income." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It said Wisconsin tax was fully eliminated. Wisconsin AGI of $35,012.56, less the $12,067.66 standard deduction and $700 exemption, leaves $22,244.90 of taxable income, and no Wisconsin nonrefundable credit offsets the resulting $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no American Opportunity Credit, so the $843.66 on $22,244.90 of taxable income stands." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It cut Wisconsin income to $33,404 by subtracting the $628 FLSA overtime premium and the $1,800 of health premiums. Neither has a Wisconsin subtraction here. It also kept 70% of the capital gain distributions, while the correct Wisconsin AGI is $35,012.56 with a $12,067.66 deduction." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"$943 implies about $24,500 of taxable income, roughly $2,260 above the correct $22,244.90. That is consistent with keeping the $1,170 of capital gain distributions in Wisconsin income and under-applying the $12,067.66 deduction or the $700 exemption." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied a capital gain subtraction to the $1,170 of distributions, which keeps part of them in Wisconsin income. The engine leaves them out entirely, giving $35,012.56 of Wisconsin AGI and $22,244.90 of taxable income. The extra income makes its $847 overshoot $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"$860 implies about $371 more taxable income than the correct $22,244.90. That matches leaving part of the $1,170 of capital gain distributions in Wisconsin income, when Wisconsin AGI should be $35,012.56." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Its $27,443 of Wisconsin income comes from subtracting the $6,589 of employer-sponsored insurance premiums and an $1,800 health insurance deduction. Neither belongs in Wisconsin AGI, which is $35,012.56. That drove taxable income down to $13,714.96 instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of non-Schedule-D gains left out of the $35,012.56 Wisconsin AGI. With an understated deduction, $936 implies about $24,300 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains left out of the $35,012.56 Wisconsin AGI. With a deduction below $12,067.66, $920 implies about $24,000 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $27,443 of Wisconsin income comes from subtracting the $6,589 of employer-sponsored insurance premiums and an $1,800 insurance deduction. Neither belongs in Wisconsin AGI, which is $35,012.56. That left $13,769.76 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It said the standard deduction and credits fully offset the tax. The $12,067.66 phased deduction and $700 exemption leave $22,244.90 of the $35,012.56 Wisconsin AGI taxable, and no Wisconsin nonrefundable credit applies." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains, and phased the deduction to about $11,046 instead of $12,067.66. That gave $24,437 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It kept 70% of the $1,170 in Wisconsin income ($35,832 instead of $35,012.56) and cut the deduction to $10,618 instead of $12,067.66. That gave $24,514 of taxable income instead of $22,244.90, taxed with an outdated $14,320 first bracket." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $628 overtime premium from AGI and kept 70% of the $1,170 of gains, ending at $35,204 of Wisconsin income instead of $35,012.56. The roughly $350 of extra taxable income produced $859 instead of $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Its $29,594 Wisconsin AGI comes from subtracting the $6,589 of employer-sponsored insurance premiums, instead of using $35,012.56. It also skipped the $700 personal exemption, giving $16,811 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It taxed income above the first bracket at the repealed 4.65% rate instead of 4.4%. It also cut the standard deduction to about $11,000 instead of $12,067.66 and kept the $1,170 of gains in a $36,183 starting income." -us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no parseable value or explanation for state_income_tax_before_refundable_credits, so nothing could be compared with the $843.66 computed from $22,244.90 of Wisconsin taxable income." -us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It kept 70% of the $1,170 of capital gain distributions in Wisconsin income ($35,832 instead of $35,012.56) and phased the deduction to $11,868.47 instead of $12,067.66. That gave $23,263.53 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It wiped out the tax with the Wisconsin School Property Tax Credit. That credit is based on property tax or rent paid, and no rent or property tax is listed, so it is $0. The $843.66 on $22,244.90 of taxable income stands." -us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 of gains, used a $13,480 deduction instead of $12,067.66, and never subtracted the $700 exemption. That gave about $22,700 of taxable income instead of $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted half of the $6,589 employer insurance premiums from wages and used the old 3.54%/4.65% rates instead of 3.5%/4.4%. It treated the $700 exemption as a $77 credit, guessed at unphased deductions, and averaged the results, instead of using $35,012.56 Wisconsin AGI less the $12,067.66 deduction and $700 exemption." -us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It overstated Wisconsin income at $40,571 and then zeroed out its own $1,374.94 with unnamed nonrefundable credits. None apply to this filer, and the correct $22,244.90 of taxable income gives $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It used an outdated sliding-scale standard deduction ($12,760 base, 12% phase-out above $18,400), giving $10,668 instead of $12,074.61, and it never subtracted the $700 exemption. It then submitted $1,257, which contradicts its own recomputed figure of about $975." +us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It started from federal AGI of $36,183 without the 30% exclusion of the $1,170 capital gain distributions ($351). It also never deducted the $700 exemption: it taxed $24,184 instead of $23,056.95, and its first bracket ($14,680) was too low." +us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the $10,152 of interest, dividends and capital gains, used the federal standard deduction instead of Wisconsin's sliding-scale deduction, and then applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no such credit, so the $878.52 liability stands." +us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed Wisconsin taxable income was low enough to produce zero tax. In fact WI AGI is $35,831.56, including $10,152 of investment income, and after the $12,074.61 standard deduction and the $700 exemption, $23,056.95 remains taxable and generates $878.52." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It wrongly refused the 30% capital gain exclusion on the non-Schedule-D capital gain distributions, which are long-term gains and qualify. It also guessed a standard deduction of about $9,500 instead of $12,074.61, and it inflated its own computed $1,014 to $1,140 with no basis." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It phased the Wisconsin standard deduction down to about $7,700 instead of $12,074.61 and skipped the 30% capital gain exclusion. It then added unexplained adjustments that raised its $1,093 to $1,163." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It assumed the standard deduction was phased down to about $9,000 instead of $12,074.61 and taxed federal AGI without the $351 capital gain exclusion. This overstated taxable income at about $26,500 instead of $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its method was correct (30% exclusion, sliding-scale deduction, $700 exemption), but its 2026 indexed standard deduction of $12,003 was $72 below the actual $12,074.61. That left taxable income at $23,129 instead of $23,056.95 and the tax about $3.50 too high." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used 2024 parameters: a $12,230 base deduction phased out above $15,000, and a $13,810 first bracket. It also omitted the 30% capital gain exclusion, which gave taxable income of $25,795 instead of $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It taxed federal AGI without the $351 capital gain exclusion and understated the standard deduction at $11,340 instead of $12,074.61, which put taxable income at $24,143. It then subtracted a 'Married/Single credit and personal exemption credit' that does not exist for this filer." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It estimated the phased standard deduction at about $11,600 instead of $12,074.61 and applied no 30% capital gain exclusion. This put taxable income near $23,900 instead of $23,056.95, and it also used a slightly low first-bracket threshold." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied a full $13,450 standard deduction with no income phase-out and skipped the 30% capital gain exclusion. Its taxable income of $22,033 is below the correct $23,056.95, and its first bracket ($14,320) was also too low." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used 2025 rules with the repealed 3.54%/4.65% rates and an unphased $13,230 standard deduction. It omitted the $700 exemption and started from an AGI of $34,383 that wrongly removed $1,800 of income." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It applied a $14,050 standard deduction with no sliding-scale phase-out, never deducted the $700 exemption, and skipped the 30% capital gain exclusion. Its taxable income was $22,133 instead of $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It phased the standard deduction down too steeply to $9,963 instead of $12,074.61, omitted the $700 exemption and the $351 capital gain exclusion, and taxed $26,219.60 instead of $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $29,594 comes from subtracting the $6,589 employer-sponsored insurance premiums from income, but those premiums are not a reduction to the given wages. It also skipped the 30% capital gain exclusion, which understated WI AGI ($35,831.56) by about $6,240." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It treated $27,000 of wages as the entire AGI. That ignores the $926 401(k) deferral and all $10,152 of interest, dividends and capital gain distributions that make up the $35,831.56 WI AGI." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $588 matches taxable income of about $16,300, the result of wrongly removing the $6,589 employer insurance premiums from income. The correct WI taxable income is $23,056.95, giving $878.52." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums to reach a state AGI of $29,594 instead of $35,831.56, and it never deducted the $700 exemption. Its taxable income of $16,887 is far below $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $1,177 implies taxable income of about $30,000, roughly $7,000 above the correct $23,056.95. That is consistent with omitting the $351 capital gain exclusion, phasing the standard deduction far below $12,074.61 and dropping the $700 exemption." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It applied a 60% capital gain exclusion, but Wisconsin's general rate is 30%, giving $351. It also subtracted a medical expense credit that does not apply to this filer, which pushed the tax down to $543.09." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It submitted $944, but its explanation states $1,018, so the answer contradicts itself. Both figures overstate taxable income relative to $23,056.95 after the $351 exclusion, the $12,074.61 standard deduction and the $700 exemption. It also wrongly treated the exemption as a credit." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $1,113 implies taxable income of about $28,000 instead of $23,056.95. That is consistent with skipping the 30% capital gain exclusion and over-phasing the sliding-scale standard deduction well below $12,074.61." +us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It claimed the Wisconsin standard deduction is fully phased out at $36k, when it is still $12,074.61. It also omitted the $700 exemption and the $351 capital gain exclusion, so it taxed the entire $36,183 and added a 5.3% bracket that never applies here." +us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It wrongly treated the sliding-scale standard deduction as fully phased out, when $12,074.61 remains. It also skipped the $700 exemption and the 30% capital gain exclusion, taxing all $36,183." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed state treatment eliminates the tax. WI AGI of $35,831.56, less the $12,074.61 standard deduction and the $700 exemption, leaves $23,056.95 of taxable income with no offsetting nonrefundable credit, so the tax is $878.52." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no such nonrefundable credit, so the $878.52 tax on $23,056.95 of taxable income is unreduced." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $628 overtime premium and the $1,800 health insurance premiums from Wisconsin income, giving $33,404. The only Wisconsin subtraction here is the $351 capital gain exclusion, so WI AGI is $35,831.56. The model also overstated the standard deduction at $12,514." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $943 implies taxable income about $1,500 above $23,056.95. That is consistent with taxing federal AGI without the $351 capital gain exclusion and phasing the standard deduction below $12,074.61." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $847 implies taxable income about $700 below $23,056.95. That is consistent with overestimating the 2026 sliding-scale standard deduction (correct value $12,074.61) or counting the exemption twice." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $860 implies taxable income about $420 below the correct $23,056.95. That is consistent with overstating the phased standard deduction relative to $12,074.61, or with mis-set 2026 bracket thresholds." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It reached Wisconsin income of $27,443 by removing both the $6,589 employer insurance premiums and the $1,800 other health premiums. Neither reduces WI AGI here; the only subtraction is the $351 capital gain exclusion, giving $35,831.56." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed about $36,183 of state income and skipped the 30% exclusion ($351) of the capital gain distributions. Combined with a low standard deduction estimate, this overstated taxable income relative to $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It used federal AGI of $36,183 as Wisconsin income with no $351 capital gain exclusion, and it phased the standard deduction below $12,074.61, which put the tax at about $920." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It reduced Wisconsin income to $27,443 by subtracting the $6,589 ESI premiums and the $1,800 insurance premiums. Neither is subtracted from WI AGI here, which is $35,831.56 after only the $351 capital gain exclusion." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and credits fully offset the tax. The $12,074.61 standard deduction and the $700 exemption leave $23,056.95 taxable, and no Wisconsin nonrefundable credit applies, so the tax is $878.52." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It started from federal AGI of $36,183 without the $351 capital gain exclusion and phased the standard deduction down to about $11,046 instead of $12,074.61. That gave taxable income of $24,437 instead of $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly reached WI AGI of $35,832 but phased the standard deduction down to $10,618 instead of $12,074.61, using too low a phase-out threshold. It also used a $14,320 first bracket instead of the higher 2026 threshold." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It reduced federal AGI to $35,555 by subtracting the $628 overtime premium. That federal overtime deduction is not an above-the-line adjustment and does not lower Wisconsin income, so WI AGI is $35,831.56." +us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used an AGI of $29,594, which wrongly removes the $6,589 employer insurance premiums from income. It also omitted the $700 exemption and the $351 capital gain exclusion." +us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied the outdated 4.65% second-bracket rate instead of 4.4% and understated the standard deduction at about $11,000 instead of $12,074.61. Its taxable income of about $24,500 is above the correct $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $878.52 reference." +us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its method was correct (30% exclusion, sliding scale, $700 exemption), but it computed the phased standard deduction as $11,868.47 instead of $12,074.61. That left taxable income $206 too high ($23,263.53) and the tax about $8.60 too high." +us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It applied the School Property Tax Credit, which requires rent or property taxes paid. This household reports neither, so no credit offsets the $878.52 tax." +us,scenario_091,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It used taxable income of about $23,660, roughly $600 above the correct $23,056.95. That is consistent with omitting the $351 capital gain exclusion and understating the sliding standard deduction relative to $12,074.61." +us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It set the phase-out threshold far too high at $27,800, which inflated the standard deduction to $13,480. It also skipped the $351 capital gain exclusion and never deducted the $700 exemption; these errors partly offset and gave $22,700 of taxable income instead of $23,056.95." +us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted half the ESI premiums ($3,294.50) from wages, used unphased standard deduction guesses and the repealed 3.54%/4.65% rates, and treated the $700 exemption as a $77 credit. The exemption is a deduction from income." +us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It overstated income at $40,571, ignoring the 401(k) deferral and the capital gain exclusion, and it skipped the sliding-scale phase-out and the exemption. It then zeroed out the tax with nonrefundable credits that this filer does not qualify for." us,scenario_091,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,400 Social Security benefit as gross income instead of applying the provisional-income formula, under which none of it is taxable. With AGI of $14,980, the aged-single standard deduction eliminates all taxable income." us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of Social Security by comparing total income directly with the $34,000 threshold; Social Security taxation uses provisional income, including only one-half of benefits, and yields no taxable Social Security here. It also inconsistently applied the basic and age-based standard-deduction amounts, but the correct $14,980 AGI is fully offset by the applicable deduction." @@ -7709,52 +7804,53 @@ us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disab us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model recited the correct statutory rule — 24 months of SSDI receipt confers Medicare — then discarded it, concluding that ""for tax-benefit modeling purposes, a disabled individual is treated as Medicare-eligible."" PolicyEngine's is_medicare_eligible implements no such shortcut: it is the age >= 65 test alone, which Dependent 1 at age 27 fails." us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model stated a disjunctive engine rule — ""In PolicyEngine, Medicare eligibility is determined by age 65 or older OR disability status"" — and fired the second branch off the is_disabled flag. That OR branch does not exist in is_medicare_eligible, which evaluates age >= 65 only, so Dependent 1 at 27 is not eligible." us,scenario_093,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed eligibility ""under the modeled disability-based Medicare eligibility criterion because disability is reported,"" attributing a disability criterion to PolicyEngine that is_medicare_eligible does not implement. The only modeled criterion is age >= 65, which Dependent 1 at age 27 fails, and the reported disability flag carries no SSDI entitlement, ESRD, or ALS status on these facts." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly got AGI of $112,408 and the $32,200 standard deduction. It then subtracted $1,000 under the new auto loan interest deduction, which requires a qualifying new vehicle assembled in the US and doesn't apply here. That cut taxable income to $79,208 instead of $80,208. It then made an unexplained further cut from $8,509 to $8,067 instead of reporting $9,129 − $500 = $8,629." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the dependent's $45,000 wages to the joint AGI, which pushed taxable income into the 22% bracket. The joint return counts only head and spouse income. It also took a $1,000 auto loan interest deduction that doesn't apply. Correct taxable income is $80,208, giving $9,129 − $500 = $8,629." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's $45,000 wages in AGI and double-counted the retirement deductions. It used an outdated $29,200 standard deduction and pre-2026 brackets instead of $32,200 and the 2026 brackets. It then subtracted an invented $4,246 for 'refundable credits and reconciliation' and never applied the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model's own reasoning put the tax at about $5,000, yet it submitted $0. Correct taxable income is $112,408 − $32,200 = $80,208, which gives $9,129 of tax and $8,629 after the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model computed AGI of $112,408 correctly but rejected the 27-year-old as a dependent using the support and gross-income tests. A permanently disabled child is a qualifying child at any age, so the $500 credit for other dependents applies. It also used the 2025 standard deduction of $31,500 and 2025 bracket edges instead of the 2026 values of $32,200 and $24,800." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the qualifying-relative gross income test to the disabled 27-year-old and so dropped the $500 credit for other dependents. That test doesn't apply to a disabled qualifying child. It also used a $31,500 standard deduction and a $24,000 top for the 10% bracket instead of the 2026 values of $32,200 and $24,800." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model deducted the full $3,000 of student loan interest instead of the $2,500 cap and used a $32,600 standard deduction. It then added about $2,000 for 'the disabled adult dependent's own tax'. The dependent's wages aren't taxed in this tax unit's liability, so its intermediate $8,371 should have been $8,629." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model assumed the joint return includes all three earners' wages and added the dependent's $45,000 to AGI. The joint return counts only head and spouse income. It also deducted $1,000 of auto loan interest that doesn't qualify. Correct taxable income is $80,208, not $124,208." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the qualifying-relative gross income limit to the disabled 27-year-old and so dropped the $500 credit for other dependents. A permanently disabled child is a qualifying child at any age. It also used the pre-OBBBA $30,000 standard deduction and 2025 brackets instead of $32,200 and the 2026 $24,800 10% bracket." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model got AGI of $112,408 right but used a $30,300 standard deduction instead of $32,200, giving a tax of $9,376. It then raised that to $10,800 with no stated computation and never applied the $500 credit for the disabled adult dependent." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model denied dependent status because of the 27-year-old's $45,000 earnings and so dropped the $500 credit for other dependents. A permanently disabled child is a qualifying child regardless of age or income. It also subtracted a $1,000 auto loan interest deduction that doesn't apply, which lowered taxable income to $79,208." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's $45,000 wages into the joint AGI and then treated the IRA and student loan deductions as fully phased out because of that inflated income. It used a $30,600 standard deduction and omitted the $500 other-dependent credit. Correct taxable income is $80,208." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate single-return tax of $2,354.82 on the dependent's wages to the joint liability, but the reference measures only the joint tax unit. It also subtracted the $8,389 ESI premiums from wages, though they don't reduce AGI here, and didn't net out the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $8,389 ESI premiums from each person's wages, cutting joint AGI to $95,630 instead of $112,408. It also folded the dependent's separate single-return income into the total. It never applied the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model added $2,309.98 of tax from a separate single return for the dependent to the joint liability, but only the joint unit's tax counts. It also subtracted ESI premiums from wages, giving joint AGI of $95,630 instead of $112,408." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model assumed TCJA expires in 2026 and applied personal exemptions with a $15,850 standard deduction. OBBBA made the TCJA structure permanent, with a $32,200 standard deduction for married filing jointly. It also subtracted ESI premiums from wages and added a separate tax on the dependent's income." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from household gross income of $163,156, which includes the dependent's $45,000 wages, though the joint return counts only head and spouse income. It also deducted the full $3,000 of student loan interest instead of $2,500, left out the $190 IRA deduction, and skipped the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied pre-TCJA 2026 parameters with personal exemptions. OBBBA permanently kept the TCJA brackets and set a $32,200 standard deduction for married filing jointly. It also added a separate return for the dependent instead of claiming the $500 other-dependent credit on the joint return." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model used post-TCJA-sunset personal exemptions and standard deductions instead of the OBBBA 2026 values of $32,200 and the 10%/12% brackets. It also subtracted ESI premiums to reach AGI of $95,630 and added $3,006.65 of tax from a separate return for the dependent." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The correct derivation gives $80,208 of taxable income, $9,129 of tax, and $8,629 after the $500 other-dependent credit. The model's $11,048 is about $2,400 too high, which is consistent with adding a separate tax on the dependent's $45,000 wages to the joint liability." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model summed the joint return with a separate return for Dependent 1, though only the joint tax unit's liability counts. It also subtracted ESI premiums from wages, though they don't reduce AGI. These two errors roughly offset, but the joint unit alone owes $9,129 − $500 = $8,629." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate single-return tax for Dependent 1 to the joint liability. The dependent is claimed on the joint return, and the output covers only that unit, whose $80,208 of taxable income gives $9,129 − $500 = $8,629." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's wages into household AGI, reaching about $157,598 instead of $112,408. It also applied personal exemptions from the TCJA sunset, which OBBBA prevented. Correct taxable income is $80,208 after the $32,200 standard deduction." -us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly claimed the $500 other-dependent credit on the joint return. It then also added $3,353.74 of tax from a separate return for the same dependent's wages, which isn't part of the joint unit's liability. It also used a $30,840 standard deduction instead of $32,200." -us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the dependent as self-supporting and added $3,210 of tax from a separate single return, and so dropped the $500 credit for the disabled qualifying child. It also deducted $3,000 of student loan interest instead of the $2,500 cap." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model claimed the deductions wipe out all tax. In fact, AGI of $112,408 minus the $32,200 standard deduction leaves $80,208 of taxable income, $9,129 of tax, and $8,629 after the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model mixed in itemized medical effects and other unsupported deductions, which put taxable income at roughly $57,000–61,000 instead of $80,208. The medical expenses are far below the 7.5%-of-AGI floor, and the $32,200 standard deduction is the only below-the-line deduction." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages in the joint return's box-1 wages and subtracted all three ESI premiums, reaching AGI of $132,241. The correct joint AGI is $112,408, from head and spouse income only, with ESI premiums not reducing wages." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's taxable income of $124,708 includes the dependent's $45,000 wages, which don't belong on the joint return. It also applied no nonrefundable credit, missing the $500 credit for the disabled adult dependent." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's pre-credit tax of $17,011.56 implies about $125,398 of taxable income. That figure includes the dependent's $45,000 wages and leaves out the $190 IRA deduction. The joint unit's taxable income is $80,208." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied the $500 other-dependent credit correctly. However, its $85,208 of taxable income implies a $27,200 deduction against the $112,408 AGI instead of the 2026 standard deduction of $32,200, which overstated pre-credit tax by $600." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the single household group as one joint return containing the dependent's wages and subtracted all three ESI premiums, reaching AGI of $132,241. The joint unit's AGI is $112,408, from head and spouse income only." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's joint tax of $9,128.96 matches the engine's. However, it added $3,220 of tax from a separate return for the dependent and left out the $500 credit for the disabled adult dependent claimed on the joint return." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $8,389 ESI premiums for both head and spouse from wages, cutting AGI to $95,630. In this computation ESI premiums don't reduce wages, so correct AGI is $112,408, taxable income is $80,208, and the tax is $9,129 before the $500 credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $13,980 implies about $111,600 of taxable income on the joint schedule, which is consistent with adding the dependent's wages to the head's and spouse's income. Only head and spouse income counts, which gives $80,208 of taxable income and $8,629 after the $500 credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied post-TCJA-sunset rules with personal exemptions and 15% brackets. OBBBA made the TCJA structure permanent, with the $32,200 standard deduction and 10%/12% brackets. It also added a separate $4,846 tax on the dependent's wages instead of claiming the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model used pre-TCJA 2026 parameters: a $16,600 standard deduction, $5,300 exemptions, and a 15% bracket. OBBBA kept the $32,200 standard deduction and the 12% bracket. It also added a separate $4,130 tax on the dependent's wages and omitted the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $3,224 tax on a separate single return for the dependent and omitted the $500 credit for the disabled adult dependent on the joint return. It also deducted $173 of 'overtime premium' even though no overtime pay is listed." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied post-TCJA brackets and a $30,750 standard deduction instead of OBBBA's permanent 2026 values of a $32,200 deduction and a 12% bracket up to $100,800. It also added a separate $3,820 tax on the dependent instead of claiming the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model's joint tax of $9,129 on $80,208 of taxable income is correct. However, it added $3,220 of tax from a separate single return for the dependent and skipped the $500 credit for the disabled adult dependent claimed on the joint return." -us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's $45,000 wages into joint AGI. On that inflated income it ruled the $190 IRA deduction fully phased out and also skipped the $4,058 traditional 401(k) exclusion, reaching taxable income of $129,456 instead of $80,208." -us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model reported $0 with no derivation. The joint unit actually has $80,208 of taxable income ($112,408 AGI minus the $32,200 standard deduction), which gives $9,129 of tax and $8,629 after the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model started AGI from $163,000 of wages, which includes the dependent's $45,000, though the joint return counts only head and spouse income. That pushed taxable income to $125,208 and into the 22% bracket instead of $80,208 in the 12% bracket." -us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted $16,778 of ESI premiums from wages and deducted the full $3,000 of student loan interest instead of the $2,500 cap. It dropped the $190 IRA deduction, denied the $500 credit for the disabled adult dependent, and submitted $7,625.75, which doesn't match its own subtotals." -us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages for $163,000 of total wages, deducted $3,000 of student loan interest instead of $2,500, and explicitly excluded the $500 other-dependent credit, which is a nonrefundable credit that this output subtracts. It then submitted $3,778, which contradicts its own $13,491 computation." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted a $1,000 auto-loan-interest deduction, which cuts taxable income to $79,208 instead of $80,208. The OBBBA vehicle-loan deduction requires qualifying-vehicle facts that the prompt does not give. It then cut its own $8,509 result to $8,067 with no explanation, when the correct result is $9,129 tax minus the $500 credit for other dependents, or $8,629." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the couple's joint AGI. The tax unit's gross income covers only the head's and spouse's wages and interest, so AGI is $112,408 and taxable income is $80,208. It also took an unsupported $1,000 auto-loan-interest deduction. Together these pushed taxable income into the 22% bracket." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in the joint return and used the outdated $29,200 standard deduction and old brackets instead of 2026's $32,200 deduction with the 10% bracket ending at $24,800. It then subtracted an invented $4,246 for 'refundable credits and reconciliation', even though the requested output comes before refundable credits. It also never applied the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Its own reasoning estimated about $5,000 of tax, yet it submitted 0. The joint return has $80,208 of taxable income, which produces $9,129 of tax, or $8,629 after the $500 credit for other dependents. A zero liability is impossible at that income." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled out the 27-year-old as a dependent using the qualifying-relative gross income test and a support test. Because the dependent is permanently disabled, they count as a qualifying child at any age, which earns the $500 credit for other dependents. It also used a $31,500 standard deduction and a $24,150 10% bracket limit instead of 2026's $32,200 and $24,800." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It denied the dependent using the qualifying-relative gross income limit, even though a permanently disabled child is a qualifying child at any age with no gross income test. That cost it the $500 credit for other dependents. It also used a $31,500 standard deduction and a $24,000 10% bracket limit instead of 2026's $32,200 and $24,800." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It deducted the full $3,000 of student loan interest, ignoring the $2,500 cap, and used a $32,600 standard deduction instead of $32,200. It then added an unexplained $2,000 to its own $8,371 result, labelled as the dependent's own tax. That tax is not part of the joint return's liability." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It put all three earners' wages ($163,000) on the joint return, when the dependent's $45,000 is not part of the tax unit's gross income. The correct AGI is $112,408. It also took an unsupported $1,000 auto-loan-interest deduction, which pushed taxable income to $124,208 and into the 22% bracket." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It disqualified the 27-year-old using the qualifying-relative gross income test, missing that permanent disability makes them a qualifying child at any age, which earns the $500 credit for other dependents. It also used a projected $30,000 standard deduction and a $23,850 bracket limit instead of 2026's $32,200 and $24,800." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It reached $9,376 using a $30,300 standard deduction instead of $32,200, then arbitrarily raised the answer to $10,800 with no supporting calculation. It also denied the dependent claim, so it missed the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled the 27-year-old out as a dependent because of their $45,000 of earnings. A permanently disabled child is a qualifying child regardless of age, with no gross income test, so the $500 credit for other dependents applies. It also took an unsupported $1,000 auto-loan-interest deduction, which put taxable income at $79,208 instead of $80,208." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in the joint AGI, then wrongly treated the IRA and student-loan deductions as fully phased out. At the correct joint AGI of $112,408, both deductions are fully allowed. It also used a $30,600 standard deduction instead of $32,200 and omitted the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate single return for the dependent ($2,354.82), but the output is only the joint return of the tax unit that claims that dependent. It also subtracted the employer health insurance premiums from wages, which understated the joint tax at $7,398.60 instead of $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It added up tax across a joint return and a separate single return for the dependent, instead of computing only the couple's joint return. It also subtracted $8,389 of employer health insurance premiums per person from wages, which lowered parent AGI to $95,630 instead of $112,408. It never applied the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate single-return tax of $2,309.98 for the dependent, even though the dependent is claimed in the couple's tax unit and has no separate liability in this output. It also subtracted the employer health insurance premiums from wages, which lowered joint AGI to $95,630 instead of $112,408." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired, bringing back personal exemptions and a $15,850 standard deduction. OBBBA made the TCJA structure permanent, with a 2026 joint standard deduction of $32,200. It also subtracted employer health insurance premiums from wages and added a separate return for the dependent." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It started from $163,156 of gross income, which includes the dependent's $45,000 of wages, but joint AGI counts only the head's and spouse's income. It also deducted the full $3,000 of student loan interest instead of the $2,500 cap, and dropped the $190 IRA deduction." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA parameters with personal exemptions, although OBBBA keeps the $32,200 joint standard deduction and TCJA brackets for 2026. It also had the dependent file a separate return and added that tax to the household total, instead of reporting only the joint return with the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate single return for the dependent to the joint return and used post-sunset personal exemptions instead of 2026 OBBBA law. It also subtracted employer health insurance premiums, which put joint AGI at $95,630 instead of $112,408." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning. The correct calculation is joint AGI of $112,408 minus the $32,200 standard deduction, giving $80,208 of taxable income and $9,129 of tax, then minus the $500 credit for other dependents for $8,629. Its $11,048 overstates the liability by about $2,400, which means it taxed income or omitted deductions or credits beyond what that joint calculation allows." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It combined the couple's joint return with a separate return for the dependent, even though the dependent belongs to the couple's tax unit and has no separate liability in this output. It also subtracted employer health insurance premiums from wages. Its total landing at $9,127.42 is a coincidence of those offsetting errors, not the correct $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the dependent as a separate single tax unit and added that tax to the couple's joint return. The output is only the joint return, at $9,129 minus the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent's $45,000 of wages in household AGI ($157,598) and applied personal exemptions from the pre-TCJA rules. Correct joint AGI excludes the dependent's wages ($112,408), and OBBBA's $32,200 standard deduction applies with no exemptions." +us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly claimed the disabled adult as a dependent for the $500 credit for other dependents, but then also added $3,353.74 of tax from a separate return for that dependent. The output is only the joint return. It also used a $30,840 standard deduction and a $23,850 bracket limit instead of $32,200 and $24,800." +us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate single-return tax of $3,210 for the dependent and denied the $500 credit for other dependents, even though the permanently disabled child is a qualifying dependent at any age. It also deducted the full $3,000 of student loan interest instead of the $2,500 cap, and used a $25,200 limit for the 10% bracket instead of $24,800." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed that deductions wipe out the household's tax. The joint return still has $80,208 of taxable income after the $32,200 standard deduction, which produces $9,129 of tax, or $8,629 after the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It offered no traceable calculation and referred to itemized medical effects that do not apply, since $2,102 of medical costs falls well below the 7.5%-of-AGI floor. Its $6,347 understates the correct figure. Taxable income is AGI of $112,408 minus the $32,200 standard deduction, or $80,208, which produces $9,129 of tax and $8,629 after the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It put all three members' wages on the joint return and subtracted all three $8,389 employer health insurance premiums, ending at AGI of $132,241. The tax unit's AGI includes only the head's and spouse's income and does not subtract those premiums, so it is $112,408 and taxable income is $80,208." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"Its taxable income of $124,708 includes the dependent's $45,000 of wages on the joint return, when only the head's and spouse's income counts, for $80,208 of taxable income. It also applied no credit at all, missing the $500 credit for other dependents for the disabled adult child." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It summed all household wages into the joint return, including the dependent's $45,000, which pushed taxable income above $125,000 and into the 22% bracket. The correct joint taxable income is $80,208." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"Its joint taxable income of $85,208 is $5,000 too high. The correct figure is AGI of $112,408 minus the $32,200 standard deduction, or $80,208, so its tentative tax of $9,728.96 is $600 above the correct $9,128.99. Its brackets and the $500 credit for other dependents were otherwise right." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages on the joint return and subtracted every member's $8,389 employer health insurance premium, which gave AGI of $132,241. The correct joint AGI excludes the dependent's wages and does not subtract those premiums, so it is $112,408." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint return's tax exactly right at $9,128.96. It then wrongly added $3,220 from a separate return for the dependent and denied the $500 credit for other dependents, even though the permanently disabled adult child is claimed in the couple's tax unit." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $8,389 employer health insurance premiums for the head and spouse from wages, lowering joint AGI to $95,630. Those premiums are not excluded from the reported wages, so AGI is $112,408 and taxable income is $80,208." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It based tax on total 'household wages', so the dependent's $45,000 is taxed along with the couple's income. The output covers only the head's and spouse's joint income: $80,208 of taxable income, $9,129 of tax, and $8,629 after the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied post-TCJA-sunset rules with personal exemptions and 15% brackets, although OBBBA keeps the $32,200 joint standard deduction and the 10%/12% brackets for 2026. It also added a separate single-return tax of $4,846 for the dependent, who is claimed in the couple's tax unit and earns them the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It used sunset-era parameters: a $16,600 standard deduction, $5,300 exemptions, and a 15% bracket, instead of 2026 OBBBA law. It also added a separate return for the dependent ($4,130) and skipped the $500 credit for other dependents on the joint return." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It added $3,224 from a separate single return for the dependent and denied the $500 credit for other dependents. It also deducted an invented $173 of overtime premium, although the head works 36 hours a week and has no overtime." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It applied post-TCJA brackets and a $30,750 standard deduction instead of 2026's $32,200 and 10%/12% brackets, which inflated the joint tax to $11,001. It also added a separate return for the dependent ($3,820) instead of claiming the $500 credit for other dependents on the joint return." +us,scenario_093,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the joint return correctly at $80,208 of taxable income and $9,129 of tax. It then added $3,220 from a separate single return for the dependent and missed the $500 credit for other dependents, which applies because the permanently disabled adult child is a qualifying dependent." +us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or reasoning was returned for this variable, so there is no submitted value to compare with $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It put all three members' wages on the joint return. That raised AGI to $161,656 and led it to treat the IRA deduction as phased out. With only the head's and spouse's income, AGI is $112,408, the $190 IRA deduction is fully allowed, and taxable income is $80,208." +us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning and reported zero tax. The joint return has AGI of $112,408 and $80,208 of taxable income after the $32,200 standard deduction, which produces $9,129 of tax, or $8,629 after the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the joint return at about $9,139 but added $3,225 from a separate return for the dependent. It also gave no $500 credit for other dependents, even though the permanently disabled adult child is claimed in the couple's tax unit." +us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in joint AGI ($157,408) and taxed the excess at 22%. The tax unit's AGI includes only the head's and spouse's income ($112,408), so taxable income is $80,208 and all of it stays within the 10% and 12% brackets." +us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted employer health insurance premiums from wages, deducted the full $3,000 of student loan interest instead of the $2,500 cap, dropped the IRA deduction, and denied the $500 credit for other dependents. It then submitted $7,625.75, which matches none of its own computed totals ($7,248 or $9,345)." +us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in joint AGI and deducted the full $3,000 of student loan interest. It then submitted $3,778, which contradicts its own computed liability of $13,491. The correct joint calculation is $80,208 of taxable income, $9,129 of tax, and $8,629 after the $500 credit for other dependents." us,scenario_093,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It got all six per-person components right ($3,286, $768.50, $4,030, $942.50, $2,790, $652.50), but added them to $11,469.50 instead of $12,469.50. It then submitted $12,348.76, a figure that matches neither its own sum nor its components." us,scenario_093,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its reasoning names the correct method, 7.65% employee FICA on $163,000 of combined wages with no Additional Medicare Tax. It never did the multiplication, which gives $12,469.50, and submitted 0 instead." @@ -7775,53 +7871,54 @@ us,scenario_093,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It sub us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation came back, so no answer could be scored." us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"It submitted 0 with no reasoning, which means no employee FICA at all on $163,000 of W-2 wages. All three workers owe 6.2% Social Security plus 1.45% Medicare, for a total of $12,469.50." us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed $4,054.50 + $4,972.50 + $3,442.50 = $12,469.50. It then submitted $12,553.50, so the error was in copying its own total into the answer." -us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model got roughly the right taxable base (~$79,800) but applied Missouri's schedule only once to the joint total. It should have split taxable income between the spouses by income share and taxed each share separately, which gives the lower-bracket savings twice. It also left out the 5% federal-tax deduction ($431.45) and deducted the full $3,000 of student loan interest instead of the $2,500 cap." -us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted the dependent's wages in joint Missouri AGI ($156,408 instead of $112,408.25). That pushed AGI above $125,000 and wrongly removed the 5% federal-tax deduction. It also applied the schedule once to the joint total instead of to each spouse's income-proportional share." -us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model used gross income of $119,156 without excluding 401(k) deferrals, IRA contributions or student loan interest. It used a $27,900 standard deduction instead of the 2026 MFJ $32,200. It then multiplied by a made-up 4.04% effective rate instead of applying Missouri's bracket schedule to each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model answered $0 with no working. The spouses' $112,408.25 of Missouri AGI, minus the $32,200 standard deduction and $431.45 federal-tax deduction, leaves $79,776.80 of taxable income. Taxing each spouse's share at rates up to 4.7% gives $3,388.25. The zero answer treats the couple as owing no Missouri tax." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model listed the $4,058 of 401(k) deferrals but left them out of its AGI arithmetic, overstating AGI at $116,466. It used a ~$30,000 standard deduction instead of $32,200 and skipped the 5% federal-tax deduction. It also applied the schedule once to the joint total instead of to each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model's own formula gave about $3,640 on $80,408 of taxable income, but it then inflated that to $4,126 with no basis. It also used a $31,500 standard deduction instead of $32,200, deducted $3,000 of student loan interest instead of the $2,500 cap, and dropped the 5% federal-tax deduction as 'near zero'. It never split taxable income between the spouses." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate Missouri tax for the dependent's $45,000 of wages, but the dependent is in the couple's tax unit with $0 Missouri AGI and owes nothing. Its couple-only figure (~$3,400) was close, and the extra dependent tax pushed the total to $4,180." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's wages into joint Missouri AGI ($157,408 instead of $112,408.25). That inflated taxable income to $125,208 and wrongly eliminated the 5% federal-tax deduction. It also applied the schedule once to the joint total instead of to each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $8,389 employer-sponsored insurance premiums from each spouse's wages, understating AGI at $95,130. It added a $4,400 personal exemption, which Missouri no longer allows. It deducted 100% of federal tax instead of the AGI-based percentage, and it applied a flat 4.7% instead of the graduated schedule on each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model estimated Missouri taxable income at ~$102,000, even though $112,408.25 of AGI minus the $32,200 standard deduction and the $431.45 federal-tax deduction gives $79,776.80. It then applied the top rate to that overstated base without splitting income between the spouses." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model got Missouri AGI ($112,408) and the $32,200 standard deduction right, but it omitted the 5% federal-tax deduction ($431.45) for MO AGI of $100,001–$125,000. It then applied the schedule once to the joint $80,208. Missouri splits taxable income between the spouses by income share and taxes each share separately, which roughly doubles the lower-bracket savings." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages in a $160,098 AGI and subtracted $2,100 personal exemptions for three people, which Missouri does not allow. It used a $30,600 standard deduction instead of $32,200 and a flat 4.5% instead of the graduated schedule with a 4.7% top rate." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $1,056.06 single-filer Missouri tax for the dependent, but the dependent is in the couple's unit with $0 Missouri AGI. It also used 2024 brackets and standard deductions instead of the 2026 $32,200 deduction and published 2026 schedule." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $99,241 of total taxable income includes the dependent's wages, which carry $0 Missouri AGI in this unit. It also used a 4.6% top rate instead of 4.7%, and it never split the couple's taxable income between the spouses." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's joint AGI of $95,630 is exactly $16,778 too low because it subtracted both spouses' employer-sponsored insurance premiums from wages. It then added a separate $801.81 tax for the dependent, who is in the couple's unit with $0 Missouri AGI." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model used a 4.8% top rate instead of 4.7%. It treated the federal-tax deduction as limited only by the $10,000/$5,000 caps, when at MO AGI of $100,001–$125,000 only 5% of federal tax ($431.45) is deductible. It also computed a separate single return for the dependent." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no working. Its $6,055 is about $2,670 above the $3,388.25 from taxing only the spouses' $79,776.80, which matches about $57,000 of extra income taxed at 4.7%. That pattern fits taxing the dependent's $45,000 of wages, losing the federal-tax deduction, and skipping the per-spouse split." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model estimated a 'combined' household tax at a flat ~4.5% rate, which taxes the dependent's wages even though the dependent has $0 Missouri AGI. It also never applied the 4.7%-top-rate graduated schedule to each spouse's share of the couple's $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $4,166 is about $780 above the $3,388.25 owed on the spouses' income alone. That gap matches an extra Missouri tax on the dependent's wages, but the dependent is in the couple's unit with $0 Missouri AGI." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no working. Its $5,321 is about $1,930 above $3,388.25, which matches roughly $41,000 more income taxed at 4.7%. That means the dependent's $45,000 of wages were taxed, though the dependent has $0 Missouri AGI here." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,421 is exactly 4.7% × $700 above the correct $3,388.25. This means it used the 2025 MFJ standard deduction of $31,500 instead of the 2026 amount of $32,200, while otherwise following the per-spouse computation." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model summed tax across 'household tax units', treating the dependent as a separate taxpayer on $45,000 of wages. The dependent belongs to the couple's unit with $0 Missouri AGI, and only the spouses' $79,776.80 of taxable income is taxed." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no working. Its $5,250 is about $1,860 above $3,388.25, which matches roughly $40,000 more income taxed at 4.7%. That means the dependent's wages were taxed, though the engine gives the dependent $0 Missouri AGI." -us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed the dependent on a separate single return, but the dependent has $0 Missouri AGI in the couple's unit. It used 2024 standard deductions ($29,200/$14,600) instead of $32,200, omitted the 5% federal-tax deduction, and applied a flat 4.7% instead of the graduated schedule on each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,356 tax for the dependent and applied a flat 4.7% to the couple's $79,708 with no graduated lower brackets. It omitted the 5% federal-tax deduction and deducted $3,000 of student loan interest instead of the $2,500 cap." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model claimed the standard deduction fully offsets the couple's income. But $112,408.25 of Missouri AGI minus the $32,200 deduction and $431.45 federal-tax deduction still leaves $79,776.80 taxable, which produces $3,388.25 of tax." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave an unanchored estimate of 'typical' figures instead of computing the tax. It never did the required steps: subtract the 5% federal-tax deduction ($431.45), split $79,776.80 between the spouses ($34,743.94 and $45,032.86), and tax each share separately for $1,452.33 + $1,935.91." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent's wages in a $157,598 federal AGI and subtracted $5,400 of Missouri personal and dependent exemptions, which Missouri no longer allows. It also used a 4.5% top rate instead of the 4.7% top rate in the 2026 schedule." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $124,708 of taxable income includes the dependent's ~$44,500 of wages in joint income, though the dependent has $0 Missouri AGI. With the inflated AGI it took no federal-tax deduction, and it applied the schedule once instead of to each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model concluded income was above the federal-tax-deduction range, which only happens if the dependent's wages are added. The spouses' Missouri AGI of $112,408.25 qualifies for the 5% federal-tax deduction ($431.45), and only their $79,776.80 is taxed." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted Missouri personal exemptions, which Missouri no longer allows. That and an oversized federal-tax deduction cut taxable income about $4,200 below the correct $79,776.80, where the only allowed deductions are the $32,200 standard deduction and 5% of federal tax ($431.45)." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $99,041 of taxable income implies an AGI of about $132,000, above the $125,000 cutoff, so it wrongly took no federal-tax deduction. The spouses' AGI is $112,408.25, which qualifies for the 5% deduction. It also applied the schedule once (4.7% × $99,041 − ~$180) instead of to each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,124 is $264 below the correct figure, which equals about $5,600 too little taxable income. That fits a federal-tax deduction far larger than the 5% of federal tax ($431.45) allowed at MO AGI of $100,001–$125,000." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed the 'working adult' dependent on $28,900 of taxable income, though the dependent has $0 Missouri AGI in the couple's unit. It also gave the spouses $80,208 of taxable income by omitting the 5% federal-tax deduction ($431.45), and it skipped the per-spouse split." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's implied Missouri AGI of $95,630 is $16,778 too low because it subtracted both spouses' $8,389 employer-sponsored insurance premiums from wages. The lower AGI also moved it into the 15% federal-tax deduction tier ($1,653.90) instead of the 5% tier ($431.45)." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,780 equals applying the schedule once to about $84,277 of joint taxable income (4.7% × income − ~$181). That base is $4,500 above $79,776.80, consistent with using the outdated $27,700 standard deduction. Missouri's per-spouse split of $79,776.80, with each share taxed separately, gives $3,388.25." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model's couple taxable income of $95,365 reflects subtracting only a single-filer $16,100 standard deduction instead of the MFJ $32,200. It added a separate tax for the dependent, who has $0 Missouri AGI, and used a 4.8% top rate instead of 4.7%." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model's MFJ taxable income of $95,808 is $111,908 minus a single-filer $16,100 standard deduction, when the MFJ deduction is $32,200. It also added a $1,555 tax for the dependent, who has $0 Missouri AGI in the couple's unit." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $1,196 single-return tax for the dependent, who carries $0 Missouri AGI in the couple's unit. For the spouses it omitted the 5% federal-tax deduction and applied the schedule once to $80,035 instead of to each spouse's income-proportional share." -us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,354 tax for the dependent, who has $0 Missouri AGI. It also used the outdated 4.95% top rate instead of 4.7% and ignored the 5% federal-tax deduction and the per-spouse split." -us,scenario_093,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a $1,047 single-unit tax for the dependent, but the dependent is in the couple's tax unit with $0 Missouri AGI. Only the spouses' $79,776.80 of taxable income is taxed, split by income share, which gives $3,388.25." -us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model used invented Missouri parameters: a $13,000 or $29,900 standard deduction instead of $32,200, $2,000 personal exemptions that Missouri no longer allows, and a 5.3% rate. It applied the 35% federal-tax-deduction tier, which only covers AGI up to $25,000, instead of the 5% tier for $100,001–$125,000, then raised the result by an arbitrary 'indexation'." -us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from a $161,656 household AGI that includes the dependent's wages, which pushed income past $125,000 and wrongly eliminated the federal-tax deduction. It then applied the schedule once to $129,456 instead of to each spouse's share of $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model answered $0 with no working. The spouses' $79,776.80 of Missouri taxable income, split $34,743.94 / $45,032.86 and taxed separately, produces $3,388.25. The zero answer ignores that Missouri tax is owed above the $32,200 standard deduction." -us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model put the dependent's wages into a $157,408 joint AGI and subtracted $3,600 of personal exemptions, which Missouri does not allow. It also skipped the 5% federal-tax deduction available at the spouses' true $112,408.25 AGI and applied the schedule once instead of per spouse." -us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the dependent's $45,000 to joint income, skipped 401(k), IRA and student-loan exclusions, and used a $26,600 standard deduction and a 4.95% rate. It then reported $11,145.20, which contradicts its own $6,809 estimate." -us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model's ~$124,942 base includes the dependent's wages, though only the spouses' income counts, giving $79,776.80 taxable. It also used a 4.5% top rate instead of the 4.7% top rate in the 2026 schedule." +us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"Got the couple's AGI (~$112,100) and standard deduction roughly right but never took Missouri's federal-income-tax deduction (5% of $8,629 = $431.45 at MO AGI $100k–$125k). It ran the whole ~$79,800 joint taxable income through the rate schedule once. Missouri splits combined-return taxable income between spouses by income share and taxes each share separately, giving $1,452.33 + $1,935.91." +us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"Put the dependent's $45,000 wages into Missouri AGI ($156,408 instead of the spouses' $112,408.25). That pushed AGI over $125,000 and zeroed the federal-tax deduction, which is actually $431.45 in the 5% tier. It also taxed the inflated $124,208 on one schedule instead of taxing each spouse's income-share separately." +us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"Started from $119,156 without excluding the $4,058 traditional 401(k) deferrals, the $190 IRA deduction or the $2,500 student-loan interest deduction. It used a $27,900 standard deduction instead of 2026's $32,200 and applied an invented 4.04% effective rate instead of Missouri's graduated schedule. It also skipped the $431.45 federal-tax deduction and the per-spouse split of the $79,776.80 taxable income." +us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Returned $0 with no computation. The couple has $79,776.80 of Missouri taxable income ($112,408.25 MO AGI − $32,200 standard deduction − $431.45 federal-tax deduction), and no nonrefundable credit offsets the resulting $3,388.25. The model treated a fully taxable middle-income joint return as having no liability." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"Left the $4,058 of traditional 401(k) deferrals in AGI ($116,466 instead of $112,408.25) and used a ~$30,000 standard deduction instead of $32,200, overstating taxable income at $86,466. It also omitted the $431.45 federal-tax deduction and taxed joint income on one schedule rather than each spouse's income-proportional share separately. Its $3,567 also falls below what its own single-schedule base produces." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"Deducted the full $3,000 of student-loan interest (the cap is $2,500), used $31,500 instead of the $32,200 standard deduction, and called the federal-tax deduction near zero when it is 5% of $8,629 = $431.45. Its own formula gives about $3,640, but it then 'adjusted' to $4,126. It also never split taxable income between the spouses for separate rate application." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"Added about $1,400 of separate Missouri tax on the dependent's $45,000 wages, but the dependent belongs to the couple's tax unit and has $0 Missouri AGI. On the couple's side it used a $32,600 standard deduction, skipped the $431.45 federal-tax deduction and did not tax each spouse's share separately." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Computed Missouri AGI as $157,408 by including the dependent's $45,000 wages; the Missouri return counts only the spouses' $112,408.25. That inflated taxable income to $125,208 and wrongly removed the 5%-tier federal-tax deduction of $431.45. It also taxed the joint amount once rather than each spouse's income share separately." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"Subtracted the $16,778 of employer-sponsored insurance premiums from wages (AGI $95,130 vs. $112,408.25), used a $30,000 standard deduction, and took a $4,400 personal exemption that Missouri no longer allows. It deducted 100% of federal tax ($7,339) when Missouri allows only an AGI-tiered percentage (5% → $431.45). It then applied a flat 4.7% instead of the graduated schedule run on each spouse's share." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"Reached ~$102,000 of Missouri taxable income, about $22,000 above the correct $79,776.80, because it netted AGI against a $30,000 standard deduction incorrectly. It also skipped the $431.45 federal-tax deduction and applied the schedule once to the joint total instead of per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Correctly reached $112,408 MO AGI and the $32,200 standard deduction but omitted the federal-income-tax deduction (5% of $8,629 = $431.45). It ran the full $80,208 through the schedule once. Missouri allocates combined-return taxable income between spouses by income share and taxes each share separately, so the lower brackets apply twice; that split alone takes about $181 off its $3,594." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"Used a $160,098 AGI that includes the dependent's $45,000 wages (Missouri AGI here is the spouses' $112,408.25), a $30,600 standard deduction instead of $32,200, and $6,300 of personal exemptions that Missouri repealed. It then applied a flat 4.5% instead of the graduated 2026 schedule per spouse and missed the $431.45 federal-tax deduction." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"Added a $1,056.06 separate single-return tax for the dependent, whose Missouri AGI is $0 inside the couple's tax unit. It computed the couple's $3,221.32 with 2024 standard deductions and brackets instead of the 2026 $32,200 standard deduction, the $431.45 federal-tax deduction, and the 2026 schedule applied to each spouse's share." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"Its $99,241 'total' Missouri taxable income adds the dependent's wages to the couple's income, but the Missouri return covers only the spouses, whose taxable income is $79,776.80. It also used a 4.6% top rate instead of the 2026 schedule's 4.7%, and did not apply the schedule separately to each spouse's income-proportional share." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Cut the couple's AGI to $95,630 by subtracting $16,778 of employer-sponsored insurance premiums, which are not wage exclusions here; the correct MO AGI is $112,408.25. It then added an $801.81 separate tax on the dependent, whose wages are not part of the Missouri return for this tax unit." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"Computed a separate single return for the dependent (citing a $5,000 single federal-tax-deduction cap) and added it to the couple's tax, though the dependent contributes $0 Missouri AGI in this one-unit household. It also used a 4.8% top rate instead of 4.7% and did not tax each spouse's share of the $79,776.80 separately." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"Gave no derivation. Its $6,055 is nearly double the correct figure and matches taxing a base that includes the dependent's $45,000 wages at roughly the top rate. The Missouri base is only the spouses' $79,776.80, split by income share and taxed per spouse on the graduated schedule." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"Taxed 'combined' household income, including the dependent's $45,000 wages, at a rough 4.5% rate. The Missouri return covers only the spouses' $112,408.25 MO AGI, minus the $32,200 standard deduction and the $431.45 federal-tax deduction. That base is taxed on the graduated 2026 schedule separately for each spouse's share." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Its $4,166 is about $778 too high, the size of a separate Missouri tax on the dependent's $45,000 wages; the dependent's Missouri AGI is $0 in this tax unit. Only the spouses' $79,776.80 of taxable income, split by income share and taxed per spouse, belongs in the total." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"Gave no derivation; its $5,321 matches a base that includes the dependent's $45,000 wages. The correct liability taxes only the spouses' $79,776.80 ($112,408.25 − $32,200 − $431.45 federal-tax deduction), with each spouse's income-proportional share run through the 2026 schedule separately." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Its $3,421 is $33 too high. That gap equals leaving out the $431.45 federal-tax deduction (about $20 of tax; 5% of $8,629 at MO AGI $100k–$125k) plus using 2025 instead of 2026 bracket thresholds in the per-spouse computation (about $9). Its taxable base was about $80,208 instead of $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Summed tax over several 'household tax units,' adding a separate liability for the dependent's $45,000 wages, but the dependent sits in the couple's unit with $0 Missouri AGI. The correct figure is only the spouses' tax on $79,776.80, allocated by income share and taxed per spouse ($1,452.33 + $1,935.91)." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Gave no derivation; its $5,250 corresponds to adding the dependent's $45,000 wages to the taxable base or taxing a dependent return. Missouri taxable income here is only the spouses' $79,776.80 after the $32,200 standard deduction and $431.45 federal-tax deduction, taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"Taxed the dependent as a separate single filer ($30,400 taxable), used 2024 standard deductions ($29,200 MFJ instead of $32,200), and applied a flat 4.7% to everything. Missouri's schedule is graduated and the $431.45 federal-tax deduction applies. The couple's taxable income is split by income share and taxed per spouse, with nothing owed for the dependent." +us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"Applied a flat 4.7% instead of the graduated schedule, deducted the full $3,000 of student-loan interest (the cap is $2,500), and added $1,356 for the dependent as a separate single filer. The dependent's Missouri AGI is $0 in this tax unit, the $431.45 federal-tax deduction applies, and each spouse's share of taxable income is taxed separately." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"Claimed the standard deduction and 'allowances' wipe out all Missouri tax. In fact $112,408.25 MO AGI minus the $32,200 standard deduction and the $431.45 federal-tax deduction leaves $79,776.80 taxable. That produces $3,388.25, and no nonrefundable credit offsets it." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"Gave an unsupported approximation of $3,427, $39 too high. It never computed the $431.45 federal-tax deduction (5% of $8,629 at MO AGI $100k–$125k) or applied the 2026 schedule to each spouse's income-proportional share ($34,743.94 and $45,032.86)." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Built a $157,598 federal AGI that includes the dependent's $45,000 wages. It then subtracted $5,400 of repealed Missouri exemptions and a $1,000 health-insurance subtraction that does not apply here, and used a 4.5% top rate. The Missouri return covers only the spouses' $112,408.25, with the $431.45 federal-tax deduction and per-spouse taxation under the 2026 schedule (4.7% top rate)." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"Its $124,708 of Missouri taxable income includes the dependent's $45,000 wages; the spouses' Missouri taxable income is $79,776.80. It also skipped the $431.45 federal-tax deduction and taxed the joint amount on one schedule instead of per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"Said the couple was above the federal-tax-deduction range because it folded the dependent's $45,000 wages into AGI. MO AGI is $112,408.25, which falls in the 5% tier and yields a $431.45 deduction. Its $5,720 reflects taxing about $125,000 on one schedule rather than $79,776.80 split between the spouses." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"Subtracted personal exemptions that Missouri eliminated starting in 2018, pushing taxable income below the correct $79,776.80 and landing $196 low. The correct base is $112,408.25 − $32,200 − $431.45 federal-tax deduction, taxed per spouse by income share." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"Its implied Missouri AGI of about $132,000 is overstated (the correct figure is $112,408.25), so it denied the federal-tax deduction, which is $431.45 in the 5% tier. It also subtracted $1,000 of health-insurance premiums not allowed here and taxed $99,041 on one schedule instead of $79,776.80 per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"Its $3,124 is $264 too low, equal to about $5,600 of excess deductions. With MO AGI of $112,408.25 in the $100,001–$125,000 tier, the only federal-tax deduction is 5% of the couple's $8,629 federal tax ($431.45). Taxable income must be $79,776.80, split between the spouses." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"Added tax on the dependent's ~$28,900 as a separate single return, but the dependent has $0 Missouri AGI in the couple's tax unit. For the couple it taxed $80,208 without the $431.45 federal-tax deduction and did not split taxable income between the spouses." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"Cut AGI to $95,630 by subtracting the $16,778 of employer-sponsored insurance premiums from wages; correct MO AGI is $112,408.25. The lower AGI put it in the 15% federal-tax-deduction tier ($1,653.90) instead of the 5% tier ($431.45), understating taxable income at $61,776.10 versus $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"Gave a one-line estimate; its $3,780 corresponds to running about $84,000 of joint taxable income through the schedule once. The correct base is $79,776.80 after the $32,200 standard deduction and $431.45 federal-tax deduction, split by income share and taxed separately for each spouse." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"Overstated the couple's taxable income at $95,365 (the correct figure is $79,776.80). It added a separate $1,547 tax on the dependent's income, but the dependent has $0 Missouri AGI in the couple's unit. It also used a 4.8% top rate instead of 4.7%." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"Overstated the couple's taxable income at $95,808 (the correct figure is $79,776.80 after the $32,200 standard deduction and $431.45 federal-tax deduction). It added $1,555 for the dependent as a separate single filer, though the dependent contributes $0 Missouri AGI, and taxed joint income on one schedule rather than per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"Added a $1,196 separate tax for the dependent, whose Missouri AGI is $0 within the couple's unit. For the couple it skipped the $431.45 federal-tax deduction and applied the schedule once to $80,035 instead of taxing each spouse's income-proportional share separately." +us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"Used an outdated 4.95% top rate (the 2026 schedule tops out at 4.7%) and added a $1,354 separate tax for the dependent, who has $0 Missouri AGI in this unit. It also left out the $431.45 federal-tax deduction and the per-spouse split of taxable income." +us,scenario_093,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"Added a $1,047 separate single-return tax for the dependent, but the dependent's Missouri AGI is $0 in the couple's tax unit. Only the couple's tax, $3,388.25 computed per spouse on $79,776.80, belongs in the total." +us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"Used invented Missouri parameters: a $13,000 (later $29,900) MFJ standard deduction, repealed $2,000 personal exemptions, a 35% federal-tax deduction, and a 5.3%–5.5% bracket schedule. It then added an arbitrary 'indexation' bump. The correct figures are a $32,200 standard deduction, a 5% federal-tax deduction of $431.45, and the 2026 schedule (4.7% top rate) applied to each spouse's share." +us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"Started from a $161,656 household AGI that includes the dependent's $45,000 wages and skips the $4,058 401(k) and $190 IRA reductions. That wiped out the federal-tax deduction and led it to tax $129,456. Missouri AGI is the spouses' $112,408.25, giving $79,776.80 of taxable income split between them." +us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"Returned $0 with no reasoning. The couple has $79,776.80 of Missouri taxable income after the $32,200 standard deduction and $431.45 federal-tax deduction, which produces $3,388.25 with no offsetting nonrefundable credit." +us,scenario_093,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"Added a $1,251 tax on the dependent's ~$28,900 as a separate filer, though the dependent has $0 Missouri AGI in the couple's unit. For the couple it left out the $431.45 federal-tax deduction and taxed $80,208 on one schedule rather than per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"Included the dependent's $45,000 wages in AGI ($157,408 instead of $112,408.25), subtracted $3,600 of personal exemptions that Missouri repealed, and taxed the joint amount once rather than per spouse. The inflated AGI also wrongly eliminated the 5%-tier federal-tax deduction of $431.45." +us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"Pooled all three people's wages with no 401(k), IRA or student-loan reductions ($164,156), used a $26,600 standard deduction and applied a flat 4.95% rate. It then reported $11,145.20, which contradicts its own $6,809 intermediate. The correct base is only the spouses' $79,776.80, taxed per spouse on the 2026 schedule (4.7% top rate)." +us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"Its $124,942 taxable base includes the dependent's $45,000 wages; the spouses' Missouri taxable income is $79,776.80. It also applied a flat 4.5% instead of the graduated 2026 schedule run separately on each spouse's income-proportional share." us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security in AGI and then applied an age-65 single-filer standard deduction. Social Security is not taxable at this provisional-income level, and the surviving-spouse filing-status deduction eliminates the resulting $15,286.89 AGI." us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model correctly reached approximately $15,286 of AGI but treated the head as a single filer and applied a $9,800 deduction. The surviving-spouse filing status provides the larger applicable standard deduction, reducing taxable income to zero." us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that none of the Social Security is taxable, but then retained it in its $29,308 AGI and computed taxable income from that inflated figure. Excluding the nontaxable Social Security produces $15,286.89 of AGI, which the surviving-spouse standard deduction fully offsets." @@ -7850,28 +7947,29 @@ us,scenario_095,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model i us,scenario_095,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an incorrect 200%-of-FPL broad-based categorical eligibility threshold and then invented a $95 monthly New Jersey minimum SNAP benefit. The household’s $29,605 income exceeds the applicable gross-income limit, and the minimum allotment applies only to an otherwise eligible household." us,scenario_095,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the household’s countable income “very low” without reconciling that claim with the listed $29,605 annual income. That income exceeds the applicable one-person gross-income limit, so the $500 asset balance cannot establish eligibility and the correct allotment calculation yields $0." us,scenario_095,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly treated $2,467 in monthly gross income as below New Jersey’s applicable broad-based categorical eligibility limit. Because the household fails the gross-income screen, it is ineligible and cannot receive the $24 monthly minimum allotment, which is reserved for otherwise eligible households." -us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly excluded Social Security and the full $11,385 401(k) distribution, leaving $4,198 of NJ gross income. It then subtracted $2,000 of exemptions and taxed $2,198 at 1.4% to get $31. It never applied New Jersey's no-tax rule: NJ gross income of $4,198 is below the $10,000 single / $20,000 qualifying-widow(er) threshold, so the tax is $0." -us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"After the full 401(k) exclusion it taxed the remaining $4,198 of self-employment income, less an $88 medical deduction and only a $1,000 exemption (it dropped the age-65 exemption), at 1.4% to get $44. It missed that NJ gross income of $4,198 after the pension exclusion is under New Jersey's $10,000/$20,000 threshold, below which no tax is owed." -us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It started from federal AGI of $15,286, deducting half of SE tax, which is not an NJ deduction. It never applied the age-62+ pension exclusion to the $11,385 401(k) distribution and taxed $13,092 at 1.4% to get $183.29. With the exclusion applied, NJ gross income is $4,198. Even without it, $15,583 is under the $20,000 qualifying-widow(er) no-tax threshold, so the tax is $0." -us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It excluded Social Security and the 401(k) distribution, then taxed $4,198 less $2,000 of exemptions at 1.4% to get $30.77. It never applied New Jersey's rule that NJ gross income at or below $10,000 (single) or $20,000 (qualifying widow(er)) owes no tax, and $4,198 is below both." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the pension exclusion, the $2,000 of exemptions and a $188 medical deduction, then taxed $2,010 at 1.4% to get $28.14. It skipped the NJ gross-income threshold: after the pension exclusion, NJ gross income is $4,198. That is below the $10,000/$20,000 no-tax threshold, so the liability is $0." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It explicitly found only $4,198 of NJ gross income left after excluding Social Security and the 401(k) distribution. It still subtracted $2,000 of exemptions and taxed $2,198 at 1.4% to get $30.77. It missed that NJ gross income that low, under $10,000 single or $20,000 qualifying widow(er), carries zero New Jersey tax." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It excluded the full $11,385 pension, took $2,000 of exemptions and taxed $2,198 at 1.4% to get $30.77. It never applied New Jersey's no-tax threshold for NJ gross income at or below $10,000/$20,000, and the post-exclusion $4,198 is below it." -us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly excluded Social Security and the 401(k) distribution, then taxed $4,198 less $2,000 of exemptions at the 1.4% bracket to get $30.77. It left out New Jersey's rule that NJ gross income of $10,000 or less (single) or $20,000 or less (qualifying widow(er)) owes no tax, so the answer is $0." -us,scenario_095,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It excluded the 401(k) distribution, then taxed roughly $3,110-$3,198 at 1.4% to get $44, having used only the $1,000 personal exemption plus a small medical deduction and omitted the age-65 exemption. It missed that NJ gross income of $4,198 after the pension exclusion is below the $10,000/$20,000 threshold, so no NJ tax is owed." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It excluded Social Security and the pension, then taxed $4,198 of net profit less $2,000 of exemptions and a $316 medical deduction at 1.4% to get $26. It never applied New Jersey's zero-tax rule for NJ gross income at or below $10,000 (single) or $20,000 (qualifying widow(er)), which covers this $4,198." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"After the retirement-income exclusion it taxed the remaining self-employment income, less exemptions and medical deductions, to get $28.14. It skipped the NJ gross-income threshold: $4,198 of post-exclusion NJ gross income is under $10,000/$20,000, so the liability before refundable credits is $0." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It excluded the retirement income, applied the regular and age-65 exemptions, and treated a 'low-income tax reduction' as only partly cutting the tax, to $18.46. New Jersey's low-income rule removes the tax entirely when NJ gross income is at or below $10,000 (single) or $20,000 (qualifying widow(er)), and $4,198 qualifies, so the tax is $0." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It excluded the $11,385 401(k) distribution, then taxed $4,198 of business income less $2,000 of exemptions at 1.4% to get $31. It never applied New Jersey's no-tax threshold: NJ gross income of $4,198 is below $10,000/$20,000, so no tax is due." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It excluded Social Security and the retirement distribution, then taxed the remaining business income, less exemptions and a medical deduction, to get $29.54. It left out New Jersey's rule that NJ gross income at or below $10,000 (single) or $20,000 (qualifying widow(er)) is not taxed, which zeroes this $4,198 of NJ gross income." -us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It correctly computed NJ gross income of $4,198, then subtracted a $316 medical deduction and $2,000 of exemptions and taxed $1,882 at 1.4% to get $26. It did not recognize that NJ gross income that low, under the $10,000 single / $20,000 qualifying-widow(er) threshold, carries no New Jersey income tax." -us,scenario_095,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It stated that NJ gross income is $4,198, then subtracted exemptions and medical expenses and taxed $1,882 at 1.4% to get $26. It never checked New Jersey's gross-income threshold, below which ($10,000 single, $20,000 qualifying widow(er)) the tax is zero, so the answer is $0." -us,scenario_095,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It computed NJ gross income of $4,198, then subtracted $2,000 of exemptions and $316.04 of medical expenses and taxed $1,881.96 at 1.4% to get $26.35. It missed that New Jersey imposes no tax when NJ gross income is $10,000 or less (single) or $20,000 or less (qualifying widow(er)), and $4,198 is below both." -us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It started from federal AGI, which wrongly subtracts half of SE tax, and took only a $1,000 exemption. It then taxed $2,901 at 1.75%, even though the first NJ bracket is 1.4%, to get $51. Beyond those errors, it missed that NJ gross income of $4,198 after the pension exclusion is below the $10,000/$20,000 threshold, below which the tax is $0." -us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It fully excluded the 401(k) distribution, took $2,000 of exemptions and a $188 medical deduction, and taxed about $2,010 at 1.4% to get $28. It never applied New Jersey's no-tax threshold for NJ gross income at or below $10,000/$20,000, which the post-exclusion $4,198 falls under." -us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score. The correct derivation excludes Social Security and the $11,385 401(k) distribution, leaving $4,198 of NJ gross income. That is below New Jersey's $10,000/$20,000 no-tax threshold, so the answer is $0." -us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It subtracted half of SE tax, which is not an NJ deduction, and took a $1,000 'standard deduction' in place of the $2,000 personal and age-65 exemptions. It then taxed $2,901 at 1.4% to get $41. It missed that NJ gross income of $4,198 after the pension exclusion is below New Jersey's $10,000/$20,000 threshold, so no tax is owed." -us,scenario_095,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It capped the retirement exclusion at $10,000 instead of fully excluding the $11,385 401(k) distribution under the age-62+ pension exclusion (worth up to $75,000 single). It then taxed $3,394.66 at 1.4% to get $47.53. Once the full exclusion is applied, NJ gross income of $4,198 is under the $10,000/$20,000 threshold, so the tax is $0." +us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly excluded Social Security and the $11,385 401(k) under the pension exclusion, leaving $4,198. It then subtracted $2,000 of exemptions and taxed $2,198 at 1.4% to get $31. It missed NJ's filing threshold ($10,000 single / $20,000 qualifying widow(er)): NJ income of $4,198 falls below it, so the tax is zero." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model applied the pension exclusion, a medical deduction and only a $1,000 exemption, then taxed $3,109.66 at 1.4% to get $44. It never applied NJ's filing threshold. With only $4,198 of NJ income left after the exclusion, below the $10,000/$20,000 threshold, the tax is zero." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model started from federal AGI and never applied NJ's age-62+ pension exclusion to the $11,385 401(k) distribution, so it taxed $13,092 at 1.4%. After that exclusion, NJ income is $4,198. That is below NJ's filing threshold, so NJ tax is zero." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model correctly reduced NJ income to $4,198 of self-employment income. It then subtracted $2,000 of exemptions and applied 1.4% to get $30.77. It missed NJ's no-tax filing threshold ($10,000 single / $20,000 qualifying widow(er)), which brings liability at this income level to zero." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly applied the pension exclusion and then took exemptions and a medical deduction, taxing $2,009.66 at 1.4%. It never checked NJ's filing threshold. NJ income of $4,198 after the exclusion is under $10,000/$20,000, so there is no NJ tax." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model correctly left only $4,198 of NJ income after the pension exclusion, then taxed $2,198 after exemptions at 1.4%. It missed that NJ charges no tax when gross income is at or below the filing threshold ($10,000 single / $20,000 qualifying widow(er)), so the answer is $0." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model fully excluded the 401(k) and then taxed $2,198 at 1.4% to get $30.77. It skipped NJ's gross-income filing threshold. The $4,198 of remaining NJ income falls far below that threshold, so liability is zero." +us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model correctly excluded Social Security and the retirement distribution, then applied $2,000 of exemptions and 1.4% to get $30.77. It never applied NJ's no-tax threshold ($10,000 single / $20,000 qualifying widow(er)), which zeroes tax on $4,198 of NJ income." +us,scenario_095,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model excluded the 401(k) and then taxed roughly $3,110–$3,198 at 1.4% to get about $44, taking only a $1,000 exemption. It ignored NJ's filing threshold. With $4,198 of NJ income, which is below $10,000/$20,000, the household owes no NJ tax." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model correctly reduced NJ income to $4,198, then subtracted exemptions and excess medical expenses and taxed the rest at 1.4% to get $26. It missed NJ's filing-threshold rule: income at or below $10,000 (single) or $20,000 (qualifying widow(er)) carries zero NJ tax." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied the retirement exclusion, exemptions and a medical deduction and taxed the remaining self-employment income to get $28.14. It never applied NJ's filing threshold, which brings tax to zero because NJ income of $4,198 is below $10,000/$20,000." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model invoked a low-income tax reduction but still reported $18.46. NJ's actual low-income rule is a full exemption: when NJ gross income is at or below the filing threshold ($10,000 single / $20,000 qualifying widow(er)), tax is zero. This household's $4,198 of NJ income qualifies." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model correctly excluded the 401(k) and then taxed $2,198 after exemptions at 1.4% to get $31. It never applied NJ's no-tax filing threshold. NJ income of $4,198 falls below that threshold, so liability is zero." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model excluded Social Security and the retirement distribution, then taxed the remaining business income after exemptions and medical expenses to get $29.54. It missed NJ's filing threshold ($10,000 single / $20,000 qualifying widow(er)), which zeroes tax on $4,198 of NJ income." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model correctly arrived at $4,198 of NJ gross income, then subtracted medical expenses and exemptions and taxed $1,882 at 1.4%. Because $4,198 is below NJ's filing threshold, NJ imposes no tax at all and the answer is $0." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model correctly found NJ gross income of $4,198, then ran it through exemptions, the medical deduction and the 1.4% bracket to get $26. It ignored the rule that NJ gross income at or below the filing threshold ($10,000 single / $20,000 qualifying widow(er)) owes zero tax." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model correctly identified NJ gross income as $4,198 but computed $26.35 by taxing $1,881.96 at 1.4%. It skipped NJ's no-tax filing threshold: gross income of $4,198 is below $10,000/$20,000, so liability is zero." +us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model started from federal AGI and applied only a $1,000 exemption. It then used a 1.75% rate instead of NJ's 1.4% first bracket, giving $51. It also missed that NJ income of $4,198 after the pension exclusion is below NJ's filing threshold, so the tax is zero." +us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model correctly applied the retirement exclusion, exemptions and a medical deduction, then taxed about $2,010 at 1.4% to get $28. It never applied NJ's filing threshold. The $4,198 of NJ income after the exclusion is below $10,000/$20,000, so tax is zero." +us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_income_tax_before_refundable_credits, so it produced no usable answer. The correct derivation gives $0 because NJ income of $4,198 after the pension exclusion is below NJ's filing threshold." +us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model subtracted half of self-employment tax, which NJ does not allow. It also used a $1,000 'standard deduction', though NJ has no standard deduction, and taxed $2,901 at 1.4% to get $41. It missed NJ's filing threshold, which zeroes tax on $4,198 of NJ income." +us,scenario_095,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"The model correctly isolated $4,198 of self-employment income after the exclusions, then taxed $2,198 after exemptions at 1.4% to get $30.77. It missed NJ's no-tax filing threshold ($10,000 single / $20,000 qualifying widow(er)), so the tax should be $0." +us,scenario_095,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model capped the retirement exclusion at $10,000, but NJ's age-62+ pension exclusion covers the full $11,385 distribution. It then taxed $3,394.66 at 1.4%. It also ignored NJ's filing threshold: NJ income after the full exclusion is $4,198, below $10,000/$20,000, so tax is zero." us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Counted the NJ ANCHOR homeowner benefit as a $1,500 refundable income tax credit. ANCHOR is a separate property-tax relief payment, not part of the NJ-1040 refundable credits (NJ EITC, NJ Child Tax Credit, property tax credit). A mortgage balance is not property tax paid, and unlisted property taxes are $0. The model also skipped the NJ EITC, which phases out to $0 at this household's AGI of about $15,286." us,scenario_095,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Correctly found the age-65+ childless NJ EITC pathway and the $298.45 phase-in amount (7.65% × $3,901 net SE earnings), but then took 40% of that without applying the phase-out. The phase-out uses the greater of AGI or earned income, and AGI is about $15,286 because it includes the $11,385 401(k) distribution. With the non-joint threshold of about $11k, the reduction of 7.65% × (AGI − threshold) exceeds $298.45, so the hypothetical federal credit and the NJ EITC are both $0." us,scenario_095,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Applied 40% to the phase-in amount alone ($298.45 = 7.65% × $3,901 of earnings after the deductible half of SE tax) and never ran the phase-out. The phase-out runs on AGI of about $15,286, which includes the $11,385 401(k) distribution. Because that distribution alone is above the roughly $11k non-joint threshold, the reduction exceeds the phase-in amount and the hypothetical federal credit, and therefore the NJ EITC, is $0." @@ -7918,143 +8016,146 @@ us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the under-five age requirement as the complete WIC eligibility test. Child 2 satisfies the categorical age rule, but household income of approximately $159,676 exceeds the 185%-of-poverty income limit, making the child ineligible." us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from child 2's age and never applied WIC's household-income test. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty limit, so the categorical age condition does not produce eligibility." us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_wic_eligible, violating the required submission contract. It therefore never reported the result produced by applying the WIC income limit: value 0." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It computed the CDCC at the old 20% rate ($1,200) instead of the OBBBA 32% rate for joint AGI of about $159.7k ($1,920). It also deducted the $144 IRA (not deductible because both spouses have workplace plans) and $1,575 of auto-loan interest (no qualifying new vehicle), and used a $32,600 standard deduction. Its submitted $12,103 does not even match its own computed $10,965." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It took only the $32,200 standard deduction and left out the OBBBA $2,000 charitable deduction for non-itemizing joint filers, which overstated taxable income by about $1,900. It also took a $144 traditional IRA deduction that the active-participant phaseout eliminates. Its credits ($6,320) were correct, so the whole error is in taxable income." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built AGI of $165,502 without removing the $3,087 of traditional 401(k) deferrals or applying the $3,000 capital-loss deduction, and used the 2024 $29,200 standard deduction. It then subtracted an invented $3,174 CTC and no CDCC, instead of the $4,400 CTC plus the $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rough $6,900 with no bracket calculation. The correct path is $17,031 of regular tax on $125,529 taxable income, minus the $4,400 CTC and $1,920 CDCC, which gives $10,711. Its answer understates tax by about $3,800, consistent with guessing at taxable income instead of applying the 2026 MFJ brackets." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used pre-OBBBA parameters: a $31,500 standard deduction, a $2,000-per-child CTC instead of $2,200, and a flat 20% CDCC instead of the 32% enhanced rate. It also left out the $2,000 non-itemizer charitable deduction and took the IRA deduction, which the active-participant phaseout eliminates. Its credits ($5,200) fell $1,120 short of $6,320." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It counted the $128 of qualified dividends twice in income and used a $31,500 standard deduction with no $2,000 charitable deduction. It also used a $2,000-per-child CTC and a 20% CDCC. It then submitted $18,962, which ignores its own net figure of $12,421 after credits." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It set the CDCC to about $0 even though $6,000 of qualifying expenses at the 32% rate gives $1,920. It used a $4,000 CTC instead of $4,400 and overstated pre-credit tax at $19,650 on about $127k of taxable income (the correct figure is about $17.0k). It also left out the $2,000 non-itemizer charitable deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $1,575 of auto-loan interest even though no qualifying new vehicle purchase is given, and missed the $2,000 OBBBA charitable deduction for non-itemizers. It also deducted the $144 traditional IRA, which the active-participant phaseout eliminates at this AGI. Together these left taxable income at $125,847 instead of $125,529.30, so tax came out about $70 too high even though the $6,320 of credits was correct." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It treated only $300 per child ($600 total) of the CTC as nonrefundable. In fact the full $2,200 per child offsets tax first, and only any excess is refundable. It also used a pre-OBBBA $30,000 standard deduction, 2025 brackets, and a 20% CDCC instead of 32%." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It estimated about $19,900 of tax before credits and then added a made-up AMT add-back to reach $24,500. AMT does not apply to this standard-deduction return, and nonrefundable credits of $6,320 must lower the $17,031 regular tax, not raise it." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It applied the CDCC at 20% ($1,200) instead of the OBBBA 32% rate ($1,920) for joint AGI just above $150,000. It also deducted $1,575 of auto-loan interest with no qualifying vehicle and left out the $2,000 non-itemizer charitable deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It counted only $600 of the CTC as nonrefundable instead of the full $4,400, and used a 20% CDCC instead of 32%. It also added the $540 short-term gain but ignored the $7,018 long-term loss, so it never took the $3,000 capital-loss deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law, with four personal exemptions and a $750 phased-out CTC. OBBBA made the TCJA structure permanent: no exemptions, a $32,200 standard deduction plus $2,000 charitable deduction, and a $2,200-per-child CTC. It also used a 20% CDCC instead of 32%." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired, taking $21,200 of personal exemptions and a $2,000 CTC. It also subtracted employer-sponsored insurance premiums from wages to reach AGI of about $135k, instead of AGI of $159,729 with the $32,200 + $2,000 deductions and the $4,400 CTC and $1,920 CDCC of permanent OBBBA law." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,700 standard deduction and no $2,000 charitable deduction, a $2,000-per-child CTC ($4,000) instead of $2,200 ($4,400), and a 20% CDCC ($1,200) instead of 32% ($1,920)." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It modeled 2026 as the post-TCJA-expiration year, with $20,200 of personal exemptions, itemized deductions, and a $1,000-per-child CTC. OBBBA made the TCJA rules permanent, so the correct path is the $32,200 standard deduction plus the $2,000 charitable deduction, the $4,400 CTC, and a 32% CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It never subtracted any nonrefundable credits; its $17,822 is about the gross regular tax. It left out the $4,400 CTC and the $1,920 CDCC, and its AGI of $164,481 also skipped the $3,000 capital-loss deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, with personal exemptions, itemizing, and a $700 phased-out CTC. It also lowered AGI to $135,169 by subtracting employer insurance premiums. Under permanent OBBBA law, AGI is $159,729, the $32,200 standard deduction plus $2,000 charitable deduction apply, and credits are a $4,400 CTC and a $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA law, with $20,200 of personal exemptions, itemized deductions, an AGI of about $135k, and a $750 phased-out CTC. It used a 20% CDCC instead of OBBBA's $32,200 + $2,000 deductions, $4,400 CTC, and 32% CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation. Its $11,463 is about $750 above the correct $10,711, which matches taking the CDCC at the old 20% rate ($1,200) instead of the OBBBA 32% rate ($1,920) that applies at joint AGI of $159,729." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $11,105 is about $394 above the correct figure, which matches leaving out the $2,000 OBBBA charitable deduction for non-itemizers (worth $440 at the 22% bracket) while otherwise applying the $32,200 standard deduction and $6,320 of credits." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It explicitly used itemized deductions and personal exemptions, which means it applied the expired pre-TCJA regime. Permanent OBBBA law for 2026 has no exemptions and uses the $32,200 standard deduction plus the $2,000 charitable deduction, with a $4,400 CTC and $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $10,817 is $106 above the correct figure, which matches about $480 of excess taxable income at the 22% bracket. That fits leaving out part of the above-the-line or non-itemizer deductions (the $337.50 educator expense and part of the $2,000 charitable deduction) while the $6,320 of credits was right." -us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It made up a $15,600 CTC instead of $4,400 and a saver's credit of $1,404, which is unavailable at AGI near $160k. It limited the CDCC to one child's $3,000 at 20% instead of $6,000 at 32%, and also ignored the 401(k) exclusion and the capital-loss deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It treated only $800 of the CTC as nonrefundable. The full $4,400 offsets tax first, since liability exceeds it. It also used a 20% CDCC instead of 32% and left out the $2,000 non-itemizer charitable deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $17,278 is roughly the gross regular tax, which means it barely applied the nonrefundable credits. The $4,400 CTC and $1,920 CDCC ($6,320 total) must be subtracted from the $17,031 regular tax." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It gave no numeric derivation. Its $14,696 is about $4,000 above the correct figure, which matches subtracting only a fraction of the $6,320 of nonrefundable credits (the full $4,400 CTC plus $1,920 CDCC) from the $17,031 regular tax." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It used the CDCC at 20% ($1,200) instead of the OBBBA 32% rate ($1,920) for joint AGI of about $159.7k. It also took only the $32,200 standard deduction without the $2,000 non-itemizer charitable deduction. Those two errors account for its $1,169 overstatement." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It subtracted a $1,200 CDCC at the old 20% rate instead of the $1,920 the OBBBA rate schedule gives (35% stepped down 1 point per $4,000 of joint AGI over $150,000, which is 32%). That understated nonrefundable credits by $720." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It took the CDCC at 20% ($1,200) instead of 32% ($1,920). Its taxable income of $127,528 also leaves out the $2,000 OBBBA charitable deduction for non-itemizers, which should bring taxable income to $125,529." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC ($4,000) instead of OBBBA's $2,200 ($4,400), and a 20% CDCC instead of 32%. It also reached taxable income of about $128k through itemizing instead of the $32,200 standard deduction plus the $2,000 charitable deduction." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It used a 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. It also overstated regular tax at about $18,360 on taxable income near $127.6k, where the 2026 MFJ brackets give about $17.5k." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It applied the CDCC at 20% ($1,200) instead of the OBBBA 32% ($1,920). Its taxable income of $127,384 also leaves out the $2,000 charitable deduction for non-itemizing joint filers." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Everything matched except the CDCC rate: it used 30% ($1,800) instead of 32% ($1,920). The OBBBA phase-down above $150,000 cuts the 35% rate by 1 point per $4,000 for joint filers, so $9,729 of excess AGI means 3 steps and a 32% rate, not 5." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave a rough $14,000 with no derivation. The correct path is $17,031 of regular tax on $125,529 taxable income (after the $32,200 standard deduction and $2,000 charitable deduction), minus the $4,400 CTC and $1,920 CDCC. Its answer overstates the result by about $3,300, consistent with under-applying those credits." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the TCJA sunset: restored 2017-style brackets, $21,200 of personal exemptions, and a CTC fully phased out. OBBBA made TCJA permanent, so the $32,200 standard deduction plus $2,000 charitable deduction, current brackets, the $4,400 CTC, and the 32% CDCC apply." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed 2026 reverted to pre-TCJA law, with personal exemptions, a $16,600 standard deduction, 10/15/25% brackets, and a $1,000 CTC phased out above $110k. Under permanent OBBBA law, the $4,400 CTC is fully available and the CDCC is 32%." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It modeled a TCJA expiration, with $5,400 personal exemptions, 10/15/25% brackets, and a fully phased-out CTC. OBBBA made the TCJA structure permanent, with a $32,200 standard deduction plus a $2,000 charitable deduction, a $4,400 CTC, and a 32% CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 10/15/25% brackets, an AGI of $134,987 that wrongly excludes employer insurance premiums, and a $751 phased-out CTC. Permanent OBBBA law gives AGI of $159,729, a $4,400 CTC, and a 32% CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It used a 20% CDCC ($1,200) instead of the OBBBA 32% rate ($1,920) and left out the $2,000 charitable deduction for non-itemizers. Its regular tax of about $17,508 is therefore too high and its credits are too low." -us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed large charitable deductions wipe out the liability. In fact itemized deductions fall below the $32,200 standard deduction, and non-itemizers get only a $2,000 charitable deduction. Taxable income of $125,529 produces $17,031 of tax, and $6,320 of credits cannot reduce that to zero." -us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used a 20% CDCC ($1,200) instead of 32% ($1,920), deducted the $144 traditional IRA despite the active-participant phaseout, and left out the $2,000 non-itemizer charitable deduction. It also used 2025 bracket boundaries and overstated regular tax at $17,852." -us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It subtracted the full $6,478 net capital loss instead of capping it at $3,000. It used a $2,000-per-child CTC and a 20% CDCC to reach $11,730, then submitted $24,711.50, which contradicts its own computation and exceeds even gross tax." -us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It never subtracted the $4,400 CTC and applied only a $1,200 CDCC. It also invented a $4,160 educator deduction, ignored the $3,000 capital-loss deduction, and used a $29,994 standard deduction instead of $32,200 plus the $2,000 charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of the OBBBA 32% rate ($1,920). It also deducted the $1,575 auto loan interest and the $144 IRA contributions, and used a $32,600 standard deduction instead of $32,200. Its own arithmetic came to $10,965, but it submitted an unrelated $12,103." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got the $32,200 standard deduction, the $4,400 CTC, and the $1,920 CDCC right. However, it left out the $2,000 OBBBA charitable deduction for non-itemizers, so taxable income came out at $127,422 instead of $125,529.30, about $417 too much tax. It also deducted the $144 IRA contributions, which the active-participant phaseout disallows." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It did not subtract the $3,088 traditional 401(k) deferrals or the $3,000 capital loss from wages, and it used the 2024 $29,200 standard deduction instead of $32,200. It then applied an invented $3,174 nonrefundable CTC and no CDCC, instead of the $4,400 CTC plus $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It gave a rough estimate with no bracket calculation. Its $6,900 implies about $10,100 in credits or extra deductions, far more than the correct $6,320 in nonrefundable credits ($4,400 CTC + $1,920 CDCC) against $17,031.48 of tax." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used pre-OBBBA parameters: a $2,000-per-child CTC instead of $2,200, a 20% CDCC rate instead of 32%, and a $31,500 standard deduction instead of $32,200. It also left out the $2,000 non-itemizer charitable deduction and taxed the qualified dividends at 0% instead of 15%." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning ended at $12,421, but it submitted $18,962. Even that reasoning counted the $128 of qualified dividends twice and used a $2,000 CTC, a 20% CDCC rate, and a $31,500 standard deduction, and it left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It gave the CDCC as roughly $0 instead of 32% × $6,000 = $1,920, and used a $4,000 CTC instead of $4,400. It also overstated tax at $19,650 on about $127,200 of taxable income (the correct brackets give about $17,500) and left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $1,575 of auto loan interest, which does not qualify, and left out the $2,000 non-itemizer charitable deduction. It also deducted the $144 IRA contributions despite the active-participant phaseout. Together these left taxable income at $125,847 instead of $125,529.30, which accounts for the $70 overstatement." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It wrongly treated only $300 per child ($600 total) of the CTC as nonrefundable. In fact the full $4,400 CTC offsets income tax liability before any refundable portion. It also used a 20% CDCC rate instead of 32%, a $30,000 standard deduction, and 2025 brackets." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It estimated about $19,900 of tax before credits and then subtracted credits, yet submitted $24,500 by citing AMT add-backs that do not apply to a standard-deduction filer. It also used a $30,300 standard deduction, a $2,000 CTC, and a 20% CDCC rate." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of the OBBBA rate of 32% at $159,729 AGI ($1,920). It also deducted $1,575 of auto loan interest where it should have taken the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $540 short-term gain but ignored the $7,018 long-term loss, so it lost the $3,000 capital loss deduction. It used a $30,900 standard deduction and limited the nonrefundable CTC to $600 instead of applying the full $4,400. It also used a 20% CDCC rate instead of 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law with $5,061 personal exemptions, itemized deductions, and a phased-down $1,000 CTC. In fact OBBBA keeps the $32,200 standard deduction, no exemptions, and a $2,200 CTC. It also subtracted the employer-paid ESI premiums from wages, which understated AGI at $134,987." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It subtracted the employer ESI premiums as Section 125 exclusions, took $21,200 of personal exemptions, and itemized under expired-TCJA rules. The correct approach is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. It also used a $2,000 CTC and a 20% CDCC rate instead of $4,400 and 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,700 standard deduction instead of $32,200, a $4,000 CTC instead of $4,400, and a 20% CDCC rate instead of 32%. It also left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied $20,200 of personal exemptions, itemized deductions, and a $1,000-per-child CTC. OBBBA made the $32,200 standard deduction and the $2,200 CTC permanent, and it also missed the 32% CDCC rate." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $164,481 leaves out the $3,000 net capital loss deduction, and it never applied the $4,400 CTC and $1,920 CDCC nonrefundable credits. Its $17,822 is essentially tax before credits." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset law with $21,200 of personal exemptions, itemized deductions, and a phased-down $700 CTC, instead of OBBBA's $32,200 standard deduction and $2,200-per-child CTC. It also subtracted the employer ESI premiums from wages, giving an AGI of $135,169." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It computed pre-TCJA tax with $20,200 of personal exemptions, itemized deductions, and a $750 phased-out CTC. Under OBBBA the household takes the $32,200 standard deduction and gets the full $4,400 CTC. Its AGI of about $135,000 also wrongly removed the employer ESI premiums from wages." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation. Its $11,463 is about $750 above the correct figure, which matches using the old 20% CDCC rate ($1,200) instead of the 32% OBBBA rate ($1,920) at $159,729 AGI." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $11,105 is $394 above the correct figure, which matches taxing the $2,000 non-itemizer charitable deduction at 22% (about $440). That deduction brings taxable income down to $125,529.30." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It said it used itemized deductions and personal exemptions, which is pre-TCJA law. For 2026, OBBBA allows no personal exemptions and gives this household the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, with a $2,200-per-child CTC." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $10,817 is $106 above the correct figure, which means it overstated taxable income by about $480 at the 22% rate. Correct taxable income is $125,529.30: AGI of $159,729.30 less the $32,200 standard deduction and the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It invented a $15,600 nonrefundable CTC instead of $4,400 and limited the CDCC to one child's $3,000 at 20% instead of $6,000 at 32%. It also claimed a $1,404 Saver's Credit, even though $159,729 AGI is far above that credit's income limit. Its deduction figures contradict each other ($30,000 vs. $18,000)." +us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It limited the nonrefundable CTC to $800, although the full $4,400 offsets income tax first. It used a 20% CDCC rate instead of 32% and left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It gave no computation. Its $17,278 is roughly the tax before any credits, so it effectively did not subtract the $4,400 CTC and $1,920 CDCC from the $17,031.48 of regular tax." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It gave no computation. Its $14,696 is consistent with subtracting only about $2,300 of credits from tax before credits, instead of the full $6,320 ($4,400 CTC plus $1,920 CDCC)." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of the OBBBA 32% rate ($1,920). It also left out the $2,000 non-itemizer charitable deduction, so taxable income was $127,566 instead of $125,529.30. These two errors account for the roughly $1,170 overstatement." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It applied a $1,200 CDCC at the old 20% rate instead of $1,920 at the OBBBA rate of 32% for $159,729 joint AGI. That understates nonrefundable credits by $720." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of 32% ($1,920). It also left out the $2,000 non-itemizer charitable deduction, so taxable income was $127,528 instead of $125,529.30." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($4,000 instead of $4,400) and a 20% CDCC rate ($1,200 instead of $1,920). It also relied on itemized deductions instead of the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. It also overstated regular tax at about $18,360, when the 2026 brackets give about $17,470 on its own taxable income." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of 32% ($1,920). It also left out the $2,000 non-itemizer charitable deduction, giving $127,384 of taxable income instead of $125,529.30." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"It got AGI, both deductions, and the CTC right but computed the CDCC as 30% ($1,800) instead of 32% ($1,920). For a joint return, the rate drops 1 point per $4,000 of AGI over $150,000, not per $2,000. That makes the $9,729 excess a 3-point reduction, not 5." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It gave a rough $14,000 guess with no bracket or credit calculation. That implies only about $3,000 of credits, far short of the $4,400 CTC plus $1,920 CDCC against $17,031.48 of tax." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied restored pre-TCJA law with personal exemptions, 10/15/25% brackets, and a $1,000 CTC fully phased out, so it allowed no CTC. Under OBBBA the household takes the $32,200 standard deduction and $2,000 charitable deduction and gets a $4,400 CTC and 32% CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied $21,200 of exemptions, itemized deductions, and 10/15/25% brackets, treating the CTC as fully phased out. OBBBA keeps the $32,200 standard deduction and a $2,200-per-child CTC with a $400,000 phaseout, and the CDCC rate is 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied post-TCJA-expiration law with $5,400 exemptions, itemizing, and restored brackets, and treated the CTC as phased out. Under OBBBA the household uses the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction and claims a $4,400 CTC and $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used TCJA-sunset rules: 10/15/25% brackets, exemptions, and a $751 phased-down CTC. It also took AGI as $134,987 by subtracting the employer ESI premiums from wages. The correct AGI is $159,729.30, with the $32,200 standard deduction and a $4,400 CTC." +us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of the OBBBA 32% rate ($1,920). It also left out the $2,000 non-itemizer charitable deduction, which lowers taxable income to $125,529.30." +us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed charitable and other deductions wipe out the tax. In fact the household takes the $32,200 standard deduction plus only a $2,000 non-itemizer charitable deduction, leaving $125,529.30 of taxable income and $17,031.48 of tax. The $6,320 of nonrefundable credits leaves $10,711.48, not $0." +us,scenario_099,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of 32% ($1,920). It also left out the $2,000 non-itemizer charitable deduction, so taxable income was $127,566 instead of $125,529.30." +us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used a 20% CDCC rate ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. It also applied 2025 bracket thresholds ($2,385 + $8,772), which overstated tax at $17,852." +us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning reached $11,730 but it submitted $24,711.50, which nothing in its work supports. That reasoning also deducted the full $6,478 capital loss instead of the $3,000 limit, and used a $2,000 CTC and a 20% CDCC rate." +us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It never applied the $4,400 CTC as a nonrefundable credit and subtracted only a $1,200 CDCC instead of $1,920. It also invented a $4,160 educator deduction, left out the $3,000 capital loss deduction, and used a $29,994 standard deduction instead of $32,200." us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Its own reasoning computed pre-credit federal tax of about $17,200 and stated that the $4,000 CTC reduces it to about $13,200, which under §24(d) leaves zero credit unused by the liability limitation, then abandoned that conclusion and reported $1,200 as an unexplained residual. It applied only the earned-income prong (15% × ($165,000 − $2,500) = $24,375) and never the controlling ceiling that the ACTC cannot exceed the CTC disallowed by tax liability. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200." us,scenario_099,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the statutory per-child refundable cap ($1,700 × 2 = $3,400) as an automatic refund for any filer with earnings above the $2,500 floor, skipping §24(d)'s requirement that the ACTC equals only the CTC remaining after the credit is applied against tax liability. With roughly $17,400 of federal income tax before credits, the full $4,400 of 2026 CTC is absorbed nonrefundably, so the refundable amount is $0. It also used the 2025 $1,700 cap instead of the 2026 indexed figure." us,scenario_099,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly identified the OBBBA 2026 $2,200 per-child CTC and the $1,800 per-child refundable cap, then treated that cap as a guaranteed refund because earnings exceed the $2,500 threshold and AGI is under the $400,000 phase-out. The refundable portion is the lesser of the earned-income formula, the per-child cap, and the CTC left over after offsetting tax; on about $127,000 of taxable income the roughly $17,400 tax consumes all $4,400 of CTC nonrefundably, leaving $0." us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so the model performed no substantive computation to evaluate. The correct derivation gives $0: about $17,400 of pre-credit federal tax fully absorbs the $4,400 CTC under the §24(d) liability limitation, and $159,676 of income is far past the two-child MFJ EITC phase-out." us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so no substantive reasoning exists to fault. The correct derivation gives $0: the roughly $17,400 of federal tax before credits leaves no CTC unused for the ACTC, and EITC is zero at $159,676 of income." us,scenario_099,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It invented a ""$300 Other Dependent Credit portion"" for two children aged 9 and 4 who are qualifying children; the §24(h)(4) other-dependent credit is $500, applies only to dependents who are not qualifying children, and is wholly nonrefundable, so it can never enter federal_refundable_credits. Its own explanation states that pre-credit tax absorbs the nonrefundable credits and no ACTC remains — a derivation that yields $0 — yet it submitted $600." -us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Compared about $166,694 of income with the federal 130% FPL free-meal cutoff and stopped there. It never applied California's universal free school meals program, which gives the 9-year-old K-12 student free meals at any income." -us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,Ruled the household out because income is above 130% FPL and it has no categorical eligibility. It missed that California's universal school meals program makes free meals available to every enrolled K-12 student with no income or categorical test. -us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Used the federal 130% FPL income test and added an asset limit that school meals do not have. It missed California's universal free school meals program, under which the 9-year-old K-12 child gets free meals regardless of income or assets." -us,scenario_099,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,Said income is far above the free-meal limit. It ignored that California's universal school meals program has no income limit and gives free meals to the enrolled K-12 child. -us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Tested income against an FPL threshold and gave it wrongly as 185%, which is the reduced-price cutoff; the free-meal cutoff is 130%. More importantly, it missed California's universal free school meals program, which makes the K-12 child eligible regardless of income." -us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"Treated the federal 130% FPL income test as the only route to free meals. California's universal meals program gives free meals to all enrolled K-12 students, which yields positive support for the 9-year-old." -us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,Concluded No based on income above 130% FPL and no categorical eligibility. It missed that California's universal free school meals program sets the meal tier to FREE for the K-12 child without regard to income. -us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"Applied only the federal 130% FPL free-meal income test. It never considered California's universal school meals program, which gives the 9-year-old free meals regardless of income." -us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Compared income with roughly $42,900, its figure for 130% FPL, under the federal NSLP rules. It missed that California's universal meals program overrides the income test and gives free meals to every K-12 student." -us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"Used only the federal 130%/185% FPL thresholds and gave their dollar amounts in reverse order. It never applied California's universal free school meals program, under which the enrolled 9-year-old gets free meals regardless of the household's $165,000 income." -us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Ruled the household ineligible because about $165k is above 130% FPL. It missed California's universal school meals program, which provides free meals to the K-12 child with no income limit." -us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Applied the federal 130% FPL free-meal limit. It missed that California's universal free school meals program makes every enrolled K-12 student eligible for free meals regardless of income. -us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Answered No only because income is above 130% FPL. It ignored California's universal school meals program, which gives the 9-year-old free meals at any income." -us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Treated income above 130% FPL as disqualifying. California's universal meals program sets the tier to FREE for all enrolled K-12 students, so the household receives positive free-meal support." -us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied only the federal 130% FPL test. It missed California's universal free school meals program, which covers the K-12 child regardless of household income." -us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Assumed an income threshold decides free-meal eligibility. In California, the universal school meals program gives free meals to every enrolled K-12 student with no income test." -us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,Used the federal 130% FPL cutoff to rule the children ineligible. It missed that California's universal meals program makes the 9-year-old K-12 student eligible for free meals regardless of income. -us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,Applied an income threshold that does not bind in California. The state's universal school meals program gives the enrolled K-12 child free meals at any income. -us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"Ruled the household out on income. It never applied California's universal free school meals program, which removes the income test for all K-12 students." -us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"Applied the federal 130% FPL free-meal guideline. It missed California's universal meals program, under which the 9-year-old receives free meals regardless of income." -us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Used an income threshold to deny free meals. California's universal school meals program gives free meals to every enrolled K-12 student, which produces positive support for this household." -us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Compared about $156,588 of AGI with 130% FPL, using AGI where the program counts gross income. More importantly, it missed California's universal free school meals program, which makes the K-12 child eligible regardless of income." -us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"Compared about $160,000 of income with a 130% FPL limit of about $43,000 under the federal rules. It ignored California's universal school meals program, which has no income test." -us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,Applied federal free-meal income thresholds. It missed that California's universal school meals program gives free meals to the enrolled 9-year-old at any income. -us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed wage income above the thresholds rules out free meals. It missed California's universal free school meals program, which makes every K-12 student eligible without an income test." -us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Relied on the federal 130% FPL limit and the absence of categorical eligibility. It never applied California's universal school meals program, which sets the meal tier to FREE for the K-12 child regardless of income." -us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"Denied eligibility on the free-meal income threshold. California's universal meals program gives free meals to all enrolled K-12 students, so PolicyEngine returns positive support." -us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied the federal income threshold. It missed California's universal free school meals program, which covers the 9-year-old K-12 student regardless of household income." -us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Used the federal free-meal income threshold. It missed that California's universal school meals program overrides it and gives free meals to every enrolled K-12 student. -us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,Treated high income as disqualifying. California's universal school meals program has no income limit and gives free meals to the enrolled K-12 child. -us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Concluded the free-meal value is zero from the 130% FPL test and the lack of categorical eligibility. It missed California's universal meals program, which gives the K-12 child $1,130.96 a year in free-meal support regardless of income." -us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Checked only the federal 130% FPL test and categorical certification through SNAP, TANF or Medicaid. It never considered California's universal free school meals program, which needs neither." -us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Compared income with a 130% FPL limit of about $43,000. It ignored that California's universal school meals program gives free meals to every enrolled K-12 student at any income." -us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"Applied the federal 130% FPL test and looked for categorical routes such as SNAP, TANF or foster care. It missed California's universal free school meals program, which makes the 9-year-old eligible regardless of income." -us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"Gave no value or explanation for free_school_meals_eligible, so there was nothing to score. The correct answer is 1, because California's universal school meals program gives free meals to the enrolled K-12 child." -us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,Applied free-meal income limits. It missed that California's universal school meals program has no income limit and covers the K-12 child. -us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Compared about $160k of income with a 130% FPL limit of about $43k. It never applied California's universal free school meals program, under which PolicyEngine sets the tier to FREE for the enrolled 9-year-old." -us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,Explicitly applied the 130% FPL threshold to a California household. It missed that California's universal school meals program gives free meals to all K-12 students regardless of income. -us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"Denied free meals on an income threshold. California's universal meals program gives free meals to every enrolled K-12 student, which produces positive support for this household." +us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"The model compared about $166,694 of income with the federal 130% free-meal cutoff (about $41,800) and stopped there. It did not apply California's universal school meals program, which gives free meals to the 9-year-old K-12 student whatever the household's income." +us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,The model treated being over 130% FPL with no categorical eligibility as the end of the analysis. It missed that California's universal free school meals program sets the tier to FREE for every enrolled K-12 student without any income test. +us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model applied the federal 130% FPL income test and also cited assets, even though school meals have no asset test. It did not recognize that California's universal school meals program gives free meals to the household's K-12 child regardless of income or assets." +us,scenario_099,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"The model ruled the household out on an income limit. In California, free school meals are universal for enrolled K-12 students and have no income limit, so the 9-year-old receives free meals." +us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model applied an FPL income test, and it misstated the free-meal cutoff as 185% (that is the reduced-price cutoff; free is 130%). It also ignored California's universal school meals program, which makes the K-12 child eligible for free meals regardless of income." +us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model used only the federal 130% FPL free-meal test. It missed that California's universal school meals program overrides that test and gives free meals to every enrolled K-12 student, including this 9-year-old." +us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model concluded 'no' because income was above 130% FPL and there was no categorical eligibility. It never applied California's universal free school meals program, which gives the FREE tier without any income or categorical test." +us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,The model ruled the household ineligible because income was above 130% FPL. It missed that California's universal school meals program gives free meals to all K-12 students regardless of income. +us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied the federal NSLP 130% FPL threshold (about $42,900) as the only route to free meals. It overlooked California's universal school meals program, which makes the 9-year-old eligible regardless of income." +us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"The model compared income with the federal 130%/185% poverty-line thresholds and stopped there. It did not account for California's universal school meals program, which gives free meals to the K-12 child without any income test." +us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,The model applied the federal 130% FPL free-meal threshold to income of about $165k. It missed that California's universal school meals program makes free meals available to every enrolled K-12 student regardless of income. +us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model used only the federal 130% FPL free-meal limit. It ignored California's universal school meals program, which gives the household's K-12 child free meals regardless of income." +us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,The model's whole test was income above 130% FPL. It missed that California's universal free school meals program sets the tier to FREE for enrolled K-12 students without any income test. +us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model denied free meals because income was above 130% FPL. In California, school meals are universal, so the 9-year-old K-12 student receives free meals regardless of income." +us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied the federal 130% FPL free-meal test. It did not apply California's universal school meals program, which overrides that test for every enrolled K-12 student." +us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model ruled the household out on an income threshold. California's universal school meals program has no income threshold, so the K-12 child receives free meals." +us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,The model applied the federal 130% FPL free-meal test. It missed that California's universal school meals program makes the household's K-12 child eligible for free meals regardless of income. +us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model treated an income threshold as decisive. In California, free school meals are universal for enrolled K-12 students and have no income test." +us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"The model ruled the household ineligible on an income threshold. It ignored California's universal school meals program, which gives free meals to the 9-year-old K-12 student regardless of income." +us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,The model applied the federal 130% poverty-guideline test. It missed that California's universal school meals program overrides that test and gives free meals to all enrolled K-12 students. +us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,The model ruled the household out on an income threshold. California's universal school meals program gives free meals to the K-12 child without any income test. +us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model compared AGI of about $156,588 with 130% FPL. It never applied California's universal school meals program, which makes the K-12 child eligible for free meals regardless of income." +us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model compared roughly $160,000 of income with a federal 130% limit of about $43,000. It missed that California's universal school meals program gives free meals to every enrolled K-12 student regardless of income." +us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,The model ruled the household ineligible on income thresholds. California's universal school meals program makes the household's K-12 child eligible for free meals regardless of income. +us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed wage income above federal thresholds rules out free meals. It overlooked California's universal school meals program, which sets the tier to FREE for enrolled K-12 students without any income test." +us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,The model applied the federal 130% poverty limit and the categorical-eligibility test. It missed that California's universal school meals program gives free meals to the 9-year-old K-12 student regardless of income or categorical status. +us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model applied an income threshold for free meals. It missed that California's universal school meals program gives free meals to all enrolled K-12 students with no income test. +us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,The model ruled the household out on an income threshold. California's universal school meals program gives the K-12 child free meals regardless of income. +us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model applied only the federal free-meal income threshold. It ignored California's universal school meals program, which overrides that threshold for every enrolled K-12 student." +us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,The model treated high income as disqualifying. California's universal school meals program gives free meals to enrolled K-12 students regardless of income. +us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model applied the federal 130% FPL test and the categorical-eligibility test. It missed that California's universal school meals program gives the household's K-12 child free meals regardless of either. +us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model checked only the federal 130% FPL limit and SNAP/TANF/Medicaid direct certification. It did not apply California's universal school meals program, which makes every enrolled K-12 student eligible for free meals." +us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model compared income with a federal 130% FPL cutoff of about $43,000. It missed that California's universal school meals program gives free meals to the 9-year-old K-12 student regardless of income." +us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model applied the federal 130% FPL test and the categorical-eligibility test (SNAP, TANF, foster care). It overlooked that California's universal school meals program makes the K-12 child eligible for free meals without either test." +us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no value or explanation for free_school_meals_eligible, so its answer could not be parsed. The correct answer is 1, because California's universal school meals program covers the household's 9-year-old K-12 student." +us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,The model ruled the household out on income limits. California's universal school meals program gives free meals to all enrolled K-12 students with no income limit. +us,scenario_099,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model applied the federal 130% poverty-guideline free-meal limit. It ignored California's universal school meals program, which gives the household's K-12 child free meals regardless of income." +us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,The model compared about $160k of income with a federal 130% FPL limit of about $43k. It missed that California's universal school meals program sets the tier to FREE for every enrolled K-12 student regardless of income. +us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,The model applied the federal 130% FPL threshold even though it noted the household was in California. It missed that California's universal school meals program gives free meals to the K-12 child regardless of income. +us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"The model ruled the household ineligible on an income threshold. California's universal school meals program has no income test, so the 9-year-old K-12 student receives free meals." us,scenario_099,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It applied CA SDI at the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). It then submitted $14,118, which does not match its own $14,602.50 total." -us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA ($12,622.50) and left out California's mandatory employee SDI. SDI is 1.3% of $165,000 in 2026, or $2,145." -us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It stated that California has no mandatory employee payroll tax. That dropped the 1.3% uncapped SDI contribution ($2,145), so it reported FICA alone at $12,622.50." -us,scenario_099,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It said California has no mandatory state payroll tax, which left out the $2,145 SDI contribution (1.3% of wages). It also submitted $12,242, which does not match its own $10,230 + $2,393 FICA sum." -us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It used the 1.2% SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145). It then submitted $13,197.50, which does not match the $14,602.50 total it computed." -us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It computed CA SDI at an outdated 1.1% rate and then left it out of the total, treating payroll_tax as federal FICA only. The prompt explicitly includes mandatory state employee payroll taxes, so the 1.3% SDI ($2,145) belongs in the total." -us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It reported only Social Security and Medicare (rounded to $12,622) and left out the mandatory CA SDI contribution of 1.3% × $165,000 = $2,145." -us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It correctly included uncapped CA SDI but used the 2024 rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145), so its total was $330 short." -us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wavered on whether to include SDI, tried it at 1.1%, and then submitted $12,903.50, a figure none of its own steps produce. The correct total adds 1.3% SDI ($2,145) to FICA of $12,622.50." -us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied CA SDI at the outdated 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $24,597 in employer-sponsored insurance premiums from the wage base, but the reference applies FICA and SDI to the full $165,000 in gross wages. It also used a 1.1% SDI rate instead of 1.3%." -us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It cut the payroll tax base to $140,403 by removing the employer-sponsored insurance premiums, when the full $165,000 in gross wages is the base. It also applied SDI at 1.1% instead of the 2026 rate of 1.3%." -us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied CA SDI at the 2024 rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It computed only federal Social Security and Medicare and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145." -us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It took the $24,597 in employer-sponsored insurance premiums out of FICA/SDI wages, but the payroll taxes are computed on the full $165,000 in gross wages. It also used 1.1% for SDI instead of the 2026 rate of 1.3%." -us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It used the outdated 1.1% CA SDI rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out the mandatory CA SDI contribution of 1.3% of wages ($2,145). Its $12,963 also does not equal 7.65% × $165,000 = $12,622.50." -us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It said it included CA SDI, but $13,280 leaves only about $657 for SDI on top of the $12,622.50 in FICA. The 2026 SDI rate is 1.3% of all $165,000 in wages, or $2,145." -us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied payroll taxes to a reduced 'taxable wages' base instead of the full $165,000 in gross wages, and it understated SDI. FICA alone on gross wages is $12,622.50, before adding $2,145 of SDI at 1.3%." -us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $14,103 implies only about $1,480 of CA SDI (roughly 0.9%) on top of $12,622.50 in FICA. The 2026 SDI rate is 1.3% with no wage cap, which gives $2,145." -us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It left out California SDI entirely, and it produced Social Security figures ($10,918.20, $10,465.20) that are not 6.2% of $165,000. The correct total is $10,230 + $2,392.50 + 1.3% SDI ($2,145)." -us,scenario_099,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It correctly treated CA SDI as mandatory and uncapped but used the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It treated the household's wages as subject to employee FICA only and left out California's mandatory 1.3% employee SDI ($2,145)." -us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It counted only FICA and left out the 1.3% CA SDI ($2,145). Its $13,577 also does not equal 7.65% × $165,000 = $12,622.50." -us,scenario_099,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It cited an approximate 1.2% SDI rate, but its $14,312.40 total implies only about $1,690 of SDI. The 2026 rate is 1.3% of $165,000, or $2,145." -us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"It used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It estimated CA SDI at about $1,980 (1.2%) instead of the 2026 amount of $2,145 at 1.3% of uncapped wages." -us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It counted only employee Social Security and Medicare (about $12,623) and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145." -us,scenario_099,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied CA SDI at the outdated 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It correctly treated CA SDI as uncapped but applied 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It reduced FICA wages to $140,403 by subtracting the employer-sponsored insurance premiums, when the full $165,000 in gross wages is the base. It also left out California's mandatory 1.3% SDI ($2,145)." -us,scenario_099,payroll_tax,inkling,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." -us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no parseable answer to score." -us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no parseable answer to score." -us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Its per-person FICA amounts ($6,502.50 + $6,120) add up to $12,622.50, but it submitted $25,290, roughly double that. It also left out the 1.3% CA SDI ($2,145)." -us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It wrongly concluded that no employee-side state payroll tax applies in California. That left out the mandatory 1.3% SDI on all wages ($2,145)." -us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It recognized that SDI counts as a mandatory state payroll tax but used an outdated 1.1% rate with a wage cap. It then left SDI out of the final $12,622.50 entirely, when uncapped 1.3% SDI ($2,145) belongs in the total." -us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied SDI only to the head's wages and miscomputed even that ($472.20 is not 1.2% of $85,000). The 2026 SDI is 1.3% on both earners' combined $165,000, or $2,145." +us,scenario_099,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Applied CA SDI at the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). Its own work then summed to $14,602.50, but it submitted $14,118, which nothing in the reasoning supports." +us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal FICA ($10,230 SS + $2,392.50 Medicare). It left out California's mandatory employee SDI of 1.3% on $165,000 ($2,145), even though the prompt explicitly includes mandatory state employee payroll taxes." +us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly stated that California has no mandatory state employee payroll tax, so it dropped the 1.3% uncapped CA SDI contribution ($2,145) and reported FICA alone ($12,622.50)." +us,scenario_099,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Said there is no mandatory CA state payroll tax and left out the 1.3% CA SDI ($2,145). It also misadded its own FICA components: $10,230 + $2,393 is $12,623, not the $12,242 it submitted." +us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Used the 2025 CA SDI rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). It then submitted $13,197.50, which does not match its own recomputed total of $14,602.50." +us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Recognized CA SDI but then dropped it, treating the output as federal FICA only. It submitted $12,622.50 without the mandatory 1.3% SDI ($2,145), and the 1.1% rate it had considered was also outdated." +us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Left out California's mandatory employee SDI (1.3% × $165,000 = $2,145) and truncated Medicare to $2,392. It reported only $12,622 of federal FICA." +us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Correctly included uncapped CA SDI but used the 2024 rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145), which left the total $330 short." +us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Went back and forth between treating CA SDI as voluntary and applying it at 1.1%, then submitted $12,903.50, which matches neither calculation. The correct addition is mandatory SDI at 1.3% of $165,000, or $2,145." +us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Applied CA SDI at the outdated 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Subtracted $24,597 of employer-sponsored insurance premiums from wages and used $140,403 as the FICA and SDI base instead of the full $165,000 of gross wages. It also applied SDI at 1.1% instead of 1.3%." +us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Cut the payroll wage base to $140,403 by subtracting ESI premiums, even though employee SS, Medicare, and SDI apply to the full $165,000 of gross wages. It also used the outdated 1.1% SDI rate instead of 1.3%." +us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied CA SDI at the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied CA SDI at the 2024 rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Computed only federal Social Security and Medicare and left out California's mandatory employee SDI (1.3% × $165,000 = $2,145)." +us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Subtracted $24,597 of ESI premiums from gross wages and taxed only $140,403, when employee FICA and SDI apply to the full $165,000. It also used the outdated 1.1% SDI rate instead of 1.3%." +us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only Social Security and Medicare and left out the mandatory 1.3% CA SDI ($2,145). Its $12,963 does not even match 7.65% × $165,000 = $12,622.50." +us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Included CA SDI but understated it badly. On top of the $12,622.50 of full FICA, its $13,280 implies only $657.50 of SDI, versus the correct 1.3% × $165,000 = $2,145." +us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,425.67 equals 8.85% (FICA plus 1.2% SDI) × $140,403. It shrank the wage base by subtracting ESI premiums and used the 2025 SDI rate. The correct calculation is 7.65% FICA plus 1.3% SDI on the full $165,000." +us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"Included CA SDI at a rate well below the 2026 1.3% rate. Its $14,103 implies about $1,480 of SDI (about 0.9%) on top of $12,622.50 of FICA, instead of $2,145." +us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"Left out California's mandatory employee SDI (1.3% × $165,000 = $2,145). It also miscomputed Social Security by applying 6.2% to wage-base figures instead of actual wages ($10,230), and ended at an unsupported $12,632.40." +us,scenario_099,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"Correctly applied uncapped CA SDI to gross wages but used the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Treated the household as subject to employee FICA only and left out California's mandatory 1.3% SDI ($2,145). It also truncated FICA to $12,622." +us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Counted only FICA and left out the mandatory 1.3% CA SDI ($2,145). Its $13,577 does not match its own FICA method, which gives 7.65% × $165,000 = $12,622.50." +us,scenario_099,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used an approximate 1.2% CA SDI rate instead of the 2026 rate of 1.3%. Its submitted $14,312.40 implies only $1,689.90 of SDI, below even its own 1.2% figure, versus the correct $2,145." +us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"Used about $1,980 of CA SDI, which is the 2025 1.2% rate, instead of the 2026 rate of 1.3% on $165,000 ($2,145)." +us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Counted only employee Social Security and Medicare and left out California's mandatory employee SDI (1.3% × $165,000 = $2,145)." +us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Applied mandatory CA SDI at the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"Applied uncapped CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted $24,597 of ESI premiums and taxed only $140,403 of FICA wages instead of the full $165,000. It also left out California's mandatory 1.3% employee SDI ($2,145) entirely." +us,scenario_099,payroll_tax,inkling,llm_error,thresholds_rates,False,"Applied CA SDI at the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no payroll_tax value or reasoning. The expected answer is $10,230 SS + $2,392.50 Medicare + $2,145 CA SDI = $14,767.50." +us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"Gave no payroll_tax value or reasoning. The expected answer is $10,230 SS + $2,392.50 Medicare + $2,145 CA SDI = $14,767.50." +us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Correctly computed per-earner FICA of $6,502.50 and $6,120 ($12,622.50 total) but submitted $25,290, roughly double that. It also left out the mandatory 1.3% CA SDI ($2,145)." +us,scenario_099,payroll_tax,muse-wonder,llm_error,thresholds_rates,False,"Applied CA employee SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Wrongly concluded that no employee-side state payroll tax applies. It left out California's mandatory SDI at 1.3% of $165,000 ($2,145)." +us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Acknowledged that the prompt includes mandatory CA SDI but still submitted FICA alone ($12,622.50). It also assumed an outdated 1.1% rate with a wage cap, where the 2026 rule is 1.3% uncapped ($2,145)." +us,scenario_099,payroll_tax,qwen3.8-max,llm_error,other,False,"Applied CA SDI only to the head's $85,000 and ignored the spouse's $80,000. It also miscomputed even that: 1.2% × $85,000 is $1,020, not $472.20. The correct SDI is 1.3% × $165,000 = $2,145." us,scenario_099,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -8064,53 +8165,52 @@ us,scenario_099,spouse_medicaid_eligible,kimi-k3,parse_contract_failure,missing_ us,scenario_099,spouse_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,spouse_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $21,275 with no 0.5%-of-AGI charitable floor, then overstated bracket tax on about $138,791 as about $6,940, when the 2026 MFJ brackets give about $5,750. Its own figure after $1,256 of exemption credits was $5,684, yet it reported $6,222, adding about $540 with no basis." -us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a medical deduction of about $4,406, but the two $2,500 premium fields describe the same premium. Correctly counted, out-of-pocket medical is at most $11,400, which is under the 7.5% floor of about $11,969, so the medical deduction is zero. That error cut taxable income to about $134,241 instead of $139,108, and it also skipped the 0.5%-of-AGI floor on charitable gifts." -us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used an outdated $4,754 CA standard deduction instead of itemizing $20,477 (real estate taxes plus charity, which beat the $11,412 standard deduction for 2026). That gave $155,605 of taxable income instead of $139,108. It then replaced the $1,284.46 personal and dependent exemption credits with a made-up $146 child credit and a made-up $543 educator-expense credit." -us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the standard deduction and worked from about $150,000 of taxable income instead of itemizing the $20,477 of real estate taxes and charity, which gives $139,108. Its rough $4,200 does not follow from any bracket calculation; the 2026 MFJ brackets give $5,778.21 on $139,108, less $1,284.46 of exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own work reached about $4,680 after exemption credits, then it raised the answer to $5,959 by 'adjusting for SDI'. CA SDI is an employee payroll tax and is excluded from state income tax. It also deducted the full $21,275 with no 0.5%-of-AGI charitable floor and guessed bracket tax well above the $5,778 the 2026 brackets give." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It got AGI of $162,672 by not subtracting the $3,088 of traditional 401(k) deferrals from wages, then added the educator expense back, pushing taxable income to $141,735 instead of $139,108. It then taxed that as if it reached the 9.3% bracket (about $7,973), when the 2026 MFJ brackets keep $139,108 in the 8% bracket at $5,778.21." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $138,000 was close, but it overstated MFJ bracket tax at about $7,700 instead of about $5,750. It then subtracted only $450 of exemption credits, even though its own components (two personal at about $150, two dependent at about $500) add up to about $1,300; the correct total is $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity, while the engine cuts the charitable deduction by 0.5% of AGI ($797.92) to $11,823.08. It also left the $338 educator expense out of CA AGI, which the engine subtracts. Together these put its taxable income at $138,647, $461 below the $139,108 that produces $5,778.21 of tax before the $1,284.46 of credits." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It overstated exemption credits at $1,800 by valuing all four exemptions at about $450. CA's personal exemption credit is only about $156 per spouse and the dependent credit about $486 each, for $1,284.46 in total. It also deducted the full $12,621 of charity with no 0.5%-of-AGI floor, giving $138,791 of taxable income instead of $139,108." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed about $150,000 of taxable income instead of $139,108 ($159,585 AGI less $20,477 itemized). It also reported its pre-credit figure unchanged and never subtracted the $1,284.46 of personal and dependent exemption credits it had listed." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added a medical deduction to reach about $25.7k of itemized deductions. The two $2,500 premium fields are the same premium, so out-of-pocket medical is at most $11,400, below the 7.5%-of-AGI floor of about $11,969, and there is no medical deduction. Itemized deductions are $20,477 and taxable income is $139,108, not about $134k." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It started from an inflated CA AGI of $163,306 instead of $159,585, failing to fully apply the 401(k), IRA and $3,000 capital-loss adjustments. It added a $2,752 medical deduction even though medical costs are under the 7.5% floor. It also subtracted only about $1,000 of dependent credits and left out the two personal exemption credits (the full total is $1,284.46)." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $112,436 implies AGI of about $135,000, which means it subtracted the $24,597 of employer-sponsored insurance premiums from wages. The listed wages are already the taxable gross and CA AGI is $159,585, so it understated taxable income by about $26,700 against $139,108." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Taxable income of $107,524 after $27,545 of itemized deductions implies AGI of about $135,069, so it removed the $24,597 of employer-sponsored insurance premiums from income and added a medical deduction of about $6,270. Neither applies: AGI is $159,585 and itemized deductions are $20,477 (real estate taxes plus charity after the floor)." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $4,395.05 medical deduction from a $16,400 medical total that counts each adult's $2,500 premium twice. Correctly counted, out-of-pocket medical is at most $11,400, under the 7.5%-of-AGI floor of about $11,969, so the medical deduction is zero and taxable income is $139,108, not $134,396." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted about $1,180 of exemption credits instead of $1,284.46 for two personal and two dependent exemptions. Its implied pre-credit tax of about $5,845 is also above the $5,778.21 the 2026 MFJ brackets produce on $139,108 of taxable income." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $7,845 is about $3,350 too high. That gap matches overstated rates plus skipping the $1,284.46 of exemption credits. The correct steps are $159,585 AGI less $20,477 itemized = $139,108 taxable income, $5,778.21 of bracket tax, minus $1,284.46 of credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It cut CA AGI to $135,169 by removing the $24,597 of employer-sponsored insurance premiums from wages. It also added a medical deduction ($26,138 itemized instead of $20,477) and subtracted only about $600 of exemption credits instead of $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It showed no figures. $2,570 plus the $1,284.46 of credits implies about $3,854 of bracket tax, which means taxable income near $114,000. That is about $25,000 below $139,108 and matches removing the $24,597 of employer-sponsored insurance premiums from wages." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $8,725 is more than the 2026 MFJ schedule charges even on the full $159,585 AGI with no deductions or credits (about $7,560), so it used inflated rates and gave no exemption credits. The correct result is $5,778.21 of tax on $139,108, minus $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $4,128 is $366 too low. At the 8% marginal rate that equals an extra deduction of about $4,500, the size of the medical deduction produced by counting the premium fields twice. In fact medical costs fall under the 7.5% floor and itemized deductions are $20,477." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,056 is $438 too low, which equals about $5,400 of extra deductions at 8%. That matches a medical deduction of about $4,400 (counting premiums twice) plus deducting full charity without the $797.92 floor. The correct itemized total is $20,477 and taxable income is $139,108." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $5,245 is $751 too high, which matches taking the $11,412 standard deduction instead of the $20,477 of itemized deductions. The extra $9,065 of taxable income at the 8% bracket adds about $725 of tax." -us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used a made-up $6,566 CA standard deduction and wrongly decided itemizing did not pay, even though real estate taxes plus charity ($20,477) exceed the $11,412 standard deduction. It then subtracted the $338 educator expense as if it were a tax credit and skipped the $1,284.46 of exemption credits. Its reported $3,202.96 also contradicts its own $5,559." -us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It took the standard deduction of about $11,500 instead of itemizing $20,477, and subtracted only about $620 of exemption credits instead of $1,284.46. It also applied a CA child and dependent care credit, which CA only allows when AGI is $100,000 or less." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation, and $6,713 is $2,219 too high. That gap matches using the standard deduction instead of the $20,477 of itemized deductions and skipping the $1,284.46 of personal and dependent exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation, and $4,931 is $437 too high. The correct steps are $159,585 AGI less $20,477 itemized = $139,108 taxable income, $5,778.21 of 2026 MFJ bracket tax, minus $1,284.46 of exemption credits; its number reflects either too-high bracket tax or too-small credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It removed employer-sponsored insurance premiums from AGI ($134,323 instead of $159,585) and subtracted exemption deductions, but CA uses exemption credits, not deductions. It then reported $6,882 of tax on $101,811, more than double the roughly $3,100 the 2026 MFJ brackets charge on that amount." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the CA standard deduction instead of itemizing. The $20,477 of real estate taxes and charity (after the floor) beats the $11,412 standard deduction, so it overstated taxable income by about $9,000." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used $160,066 of AGI, leaving out the $144 IRA and $338 educator adjustments, and deducted the full $21,275 with no 0.5%-of-AGI charitable floor. Even on its own $138,791 of taxable income, 2026 brackets less $1,284.46 of credits give about $4,468, so its $4,382 also understates bracket tax." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It gave no figures, and $4,250 is $244 too low, the same as about $3,000 of excess deductions at the 8% marginal rate. The correct steps are $139,108 of taxable income after $20,477 of itemized deductions, $5,778.21 of tax, minus $1,284.46 of exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It used 2025 MFJ brackets without indexing ($5,871.38 on $138,791) and 2025 exemption credits ($1,256) instead of the 2026 values ($5,778.21 on $139,108 and $1,284.46 of credits). It also deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor and used $160,066 of AGI instead of $159,585." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the right kinds of deduction (charity and property tax) but came out $644 too high. Its bracket tax was above the $5,778.21 the 2026 MFJ brackets give on $139,108, and its exemption credits fell short of $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted full charity without the 0.5%-of-AGI floor, so its taxable income was $138,647 instead of $139,108, yet it still came out $72 high. That is because it applied bracket thresholds below the 2026 indexed values, overstating tax on that income." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied 2025 MFJ brackets without indexing ($5,871.38 on $138,791) and the 2025 exemption credit total ($1,256) instead of the 2026 values ($5,778.21 on $139,108 and $1,284.46). It also deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess ('~165k income approx 4.5k') with no deduction, bracket or credit work. The actual steps are $159,585 AGI, $20,477 itemized, $139,108 taxable, $5,778.21 of tax and $1,284.46 of exemption credits, giving $4,493.74, so the round number misses by $6.26." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $21,275 with no 0.5%-of-AGI charitable floor, then overstated bracket tax: $5,890 on $138,453, against $5,778.21 on the correct $139,108. That leaves it $106 too high even though its credit estimate of about $1,290 was close." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor, giving $138,791 of taxable income and $5,743 of tentative tax instead of $139,108 and $5,778.21. It also understated exemption credits at $1,246 instead of $1,284.46. The two errors nearly cancel, leaving it $3.26 too high." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity without the $797.92 (0.5% of AGI) floor and used $159,766 of AGI, giving $138,491 of taxable income instead of $139,108. That $617 shortfall at the 8% rate, plus its bracket estimate, leaves the tax $67 too low." -us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It used $110,811 of taxable income, about $28,300 below the correct $139,108 ($159,585 AGI less $20,477 itemized). It also said no nonrefundable credits apply, but the $1,284.46 of personal and dependent exemption credits are available because their phase-out starts far above this AGI." -us,scenario_099,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only $576 of exemption credits instead of $1,284.46 (two personal at about $156 and two dependent at about $486 each). That shortfall alone accounts for $708 of its $796 overstatement. It also deducted full charity without the 0.5%-of-AGI floor." -us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score. The correct steps give $139,108 of taxable income, $5,778.21 of tax and $1,284.46 of exemption credits, for $4,493.74." -us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score. The correct steps give $139,108 of taxable income, $5,778.21 of tax and $1,284.46 of exemption credits, for $4,493.74." -us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported zero tax. CA taxes $139,108 of taxable income ($159,585 AGI less $20,477 itemized) at $5,778.21 before credits, and only $1,284.46 of nonrefundable exemption credits offset that." -us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It assumed CA forces the standard deduction because the federal return takes it, but CA lets filers itemize on the state return on their own. The $20,477 of real estate taxes and charity (after the floor) beats the $11,412 standard deduction, so taxable income is $139,108, not about $147,800." -us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It skipped both the $3,000 capital-loss limit and the $3,088 401(k) exclusion in AGI and said no nonrefundable credits apply, leaving out $1,284.46 of exemption credits. It also badly understated bracket tax at $4,301.50 on $138,063, where the 2026 MFJ schedule gives about $5,695. The missing credits and the understated tax roughly cancel." -us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It used a made-up $34,000 standard deduction, when CA's 2026 MFJ standard deduction is $11,412 and the $20,477 of itemized deductions applies. It also overstated tax at $8,100 on $130,894 (2026 brackets give about $5,120), applied a renter's credit that is not available at this AGI, and left out the $1,284.46 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It reached the correct taxable income of about $138,791 but overstated 2026 MFJ bracket tax at about $6,940 instead of $5,871.48. It then wrote $6,940 − $1,256 ≈ $5,684 and still submitted $6,222, a figure that does not match its own subtraction of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It claimed a ~$4,406 medical deduction by counting the $2,500 'other health insurance premiums' on top of the $2,500 premiums-excluding-Part-B figure, which already includes them. That inflated medical costs to $16,400, but actual out-of-pocket medical of $11,400 is below the 7.5% floor (~$12,005). It also took the $144 IRA deduction, which is phased out for active plan participants at this income, so taxable income came out ~$134,241 instead of $138,792." +us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used an outdated $4,754 standard deduction instead of the $11,412 MFJ amount and never itemized the $21,275 of property tax and charity. It also deducted unreimbursed employee expenses above the line. It then subtracted an invented $146 'child tax credit' and an educator 'benefit' instead of the $1,256 personal and dependent exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It took the standard deduction, giving roughly $150,000 of taxable income, instead of itemizing the $21,275 of real estate taxes and charity, which yields $138,792. Its $4,200 is a rough guess that is not tied to bracket tax of $5,871.48 less $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,payroll_tax_base,False,"It got itemized deductions right and reached a net of about $4,680. It then raised the answer to $5,959 'for SDI', but CA SDI is an employee payroll tax and is not part of state income tax before refundable credits. It also overstated exemption credits (~$1,520 vs $1,256) and bracket tax (~$6,200 vs $5,871)." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It said 401(k) deferrals were already netted out but then used gross wages of $165,000, failing to subtract the $3,088 traditional 401(k) deferrals. That put CA AGI at $163,010 and taxable income at $141,735. It also treated income as reaching the 9.3% bracket and overstated bracket tax at ~$7,973 vs the correct $5,871.48." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Taxable income of ~$138,000 was right, but it overstated MFJ bracket tax at ~$7,700 instead of $5,871.48. It also mis-added its own exemption credits (2×$150 + 2×$500 is $1,300, not $450), subtracting only $450 instead of the $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It took the $144 IRA deduction, which is fully phased out for active plan participants at ~$160k MAGI. Its projected 2026 bracket tax (~$5,729) was about $140 below the correct $5,871.48. It also overstated exemption credits at $1,288 vs $1,256, so the result was ~$175 too low." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Its CA AGI ($160,066) and taxable income ($138,791) were correct. However, it applied a ~$450 credit to each of the four people, $1,800 in total, instead of ~$153 per personal exemption and ~$475 per dependent ($1,256). That overstated exemption credits by ~$544 and drove the answer below the correct figure." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It capped real estate taxes at the federal $10,000 SALT limit, which California does not apply, and ended up with ~$150,000 of taxable income instead of $138,792. It also said exemption credits should be subtracted but left the $6,800 unchanged, so the $1,256 of exemption credits was never applied." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It added a medical deduction above the 7.5% floor, but that deduction exists only because it counted the $2,500 'other health insurance premiums' twice per adult. Actual out-of-pocket medical of $11,400 is below the ~$12,005 floor, so there is no medical deduction. As a result, taxable income was ~$134k instead of $138,792." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $163,306 is ~$3,240 too high, consistent with not excluding the $3,088 traditional 401(k) deferrals. It added a $2,752 medical deduction even though out-of-pocket medical of $11,400 is below the 7.5% floor. It also used only ~$1,000 of exemption credits instead of $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its taxable income of $112,436 implies CA AGI of about $135k, consistent with subtracting the $24,597 of employer-sponsored insurance premiums from wages, although gross wages remain fully taxable. It also added the $1,263 of unreimbursed employee expenses to itemized deductions, but those fall below the 2% AGI floor (~$3,201)." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its taxable income of $107,524 implies CA AGI of about $135k, consistent with excluding the $24,597 of ESI premiums from wages. It also inflated itemized deductions to $27,545 with non-allowable medical and employee-expense items. The correct figures are CA AGI of $160,066.80, itemized deductions of $21,274.50, and taxable income of $138,792.30." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its CA AGI ($160,066) was correct, but it claimed a $4,395.05 medical deduction by counting both the $2,500 premiums-excluding-Part-B and the $2,500 'other health insurance premiums' for each adult. True out-of-pocket medical of $11,400 is below the 7.5% floor, so taxable income is $138,792, not $134,396." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It subtracted only ~$1,180 of exemption credits instead of $1,256 (2 personal at ~$153 plus 2 dependent at ~$475). Its implied bracket tax of ~$5,845 also falls short of $5,871.48, leaving the answer $50 high." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation, and its $7,845 is ~$3,230 above the correct figure. That amount exceeds even the bracket tax on $138,792 of taxable income ($5,871.48), so it both overstated bracket tax and omitted the $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $135,169 is ~$24.9k too low, consistent with excluding the $24,597 of employer-sponsored insurance premiums from taxable wages. Its itemized deductions of $26,138 include non-allowable medical and employee-expense amounts. It also used only ~$600 of exemption credits instead of $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $2,570 is ~$2,045 below the correct figure, consistent with understating taxable income by roughly $25k, for example by excluding ESI premiums from wages or adding medical and employee-expense deductions. The correct base is $138,792.30, giving $5,871.48 of bracket tax less $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $8,725 exceeds the bracket tax on even the standard-deduction taxable income (~$148.7k), so it ignored the $21,275 of itemized deductions and the $1,256 of exemption credits and overstated the bracket schedule. The correct figure is $5,871.48 of tax on $138,792.30, less $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no derivation, and its $4,128 is ~$487 below the correct figure. That gap corresponds to an extra ~$5–6k deduction (such as a medical deduction from double-counted premiums) or an understated 2026 bracket tax. The correct figure is $5,871.48 of tax on $138,792.30 of taxable income, less $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no derivation, and its $4,056 is ~$559 below the correct figure. That gap is consistent with extra itemized deductions beyond property tax and charity, or an understated bracket tax, instead of $5,871.48 of bracket tax on $138,792.30 less $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It gave no derivation, and its $5,245 is ~$630 above the correct figure. That gap is consistent with overstated bracket tax or exemption credits well below $1,256. The correct result is $5,871.48 of tax on $138,792.30 of taxable income less $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a non-existent $6,566 MFJ standard deduction instead of $11,412 and wrongly concluded that itemized deductions did not exceed it, when $21,275 does. It treated the $338 educator expense as a dollar-for-dollar credit, although California disallows that deduction entirely. It also omitted the $1,256 of exemption credits and then submitted $3,202.96, a figure inconsistent with its own $5,559." +us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the ~$11,500 standard deduction instead of itemizing the $21,275 of real estate taxes and charity, giving ~$149k of taxable income. It used only ~$620 of exemption credits instead of $1,256. It also applied a $240 CA dependent care credit, which is unavailable because California limits that credit to AGI of $100,000 or less." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no computation, and its $6,713 is ~$2,100 above the correct figure. That result is consistent with using the standard deduction instead of the $21,275 of itemized deductions and omitting most of the $1,256 of exemption credits. The correct figure is $5,871.48 of tax on $138,792.30, less $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no computation, and its $4,931 is ~$316 above the correct figure. That is consistent with an overstated 2026 bracket tax or understated exemption credits, versus $5,871.48 of tax on $138,792.30 less $1,256 of credits." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It reduced CA AGI to $134,323 by excluding 'health-premium' amounts and taking IRA deductions, but the ESI premiums do not reduce gross wages and the IRA deduction is phased out. It then subtracted exemptions as deductions to reach $101,811, although California gives exemptions as credits. Its $6,882 is also far above the bracket tax on its own taxable-income figure." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the CA standard deduction ($11,412) instead of itemizing the $21,275 of real estate taxes ($8,654) and charity ($12,621). That overstated taxable income by about $9,860 and produced ~$5,489 instead of $4,615." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its CA AGI ($160,066) and itemized deductions were right, but its bracket and credit estimate came out ~$233 low. Either its bracket tax was below $5,871.48 on $138,792 of taxable income or its exemption credits exceeded $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It gave only a summary, and its $4,250 is ~$365 below the correct figure. That gap reflects understated 2026 bracket tax or excess deductions or credits beyond the $21,275 of itemized deductions and $1,256 of exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It itemized charity and property taxes correctly, but its $5,138 is ~$523 high. That means it overstated bracket tax relative to $5,871.48 on $138,792 or used exemption credits well below the $1,256 for two personal and two dependent credits." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It took the $144 IRA deduction, which is phased out for active plan participants at this income, giving taxable income of $138,647 instead of $138,792. Its approximate bracket and exemption-credit amounts left it $49 below the correct figure." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It guessed ~$4.5k from '~165k income' without computing CA AGI, itemized deductions, bracket tax, or exemption credits. The correct path is $138,792.30 of taxable income, $5,871.48 of tax, and $1,256 of credits." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It used federal AGI ($159,728) as CA AGI without adding back the $337.50 educator deduction that California disallows, which understated taxable income. It also overstated exemption credits at ~$1,290 vs $1,256, leaving the answer ~$15 low." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its AGI and taxable income ($138,791) were correct, but its 2026 MFJ bracket tax of ~$5,743 was ~$128 below the correct $5,871.48 because its estimated bracket thresholds were too high. It also used ~$1,246 of exemption credits instead of $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its CA AGI of $159,766 was ~$300 short of $160,066.80, so taxable income was $138,491 instead of $138,792. Its estimated 2026 bracket tax and exemption credits then came out ~$165 too low on top of that, giving $4,427." +us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It understated taxable income at $110,811 instead of $138,792, consistent with excluding ESI premiums or over-deducting. It also claimed no nonrefundable credits apply at this income, although the $1,256 personal and dependent exemption credits apply in full because CA's phase-out starts at a much higher AGI." +us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached taxable income of ~$138,491 and bracket tax near $5,866 but subtracted only $576 of exemption credits. The correct amount is $1,256: two personal credits of ~$153 each plus two dependent credits of ~$475 each." +us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits, so there is no answer to evaluate." +us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits, so there is no answer to evaluate." +us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 even though a joint household with $160,066.80 of CA AGI and $138,792.30 of taxable income owes $5,871.48 of bracket tax. Exemption credits are only $1,256, so the liability is $4,615.48." +us,scenario_099,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It used ~$149,000 of taxable income, which implies the standard deduction instead of itemizing the $21,275 of real estate taxes and charity. It also applied a ~$240 CA dependent care credit, which California denies above $100,000 AGI, and used ~$1,190 of exemption credits instead of $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It claimed California requires the standard deduction when the federal standard deduction is taken. In fact, California lets filers itemize on Schedule CA independently, and the $21,275 of real estate tax and charity exceeds the $11,412 standard deduction. It also kept the federal educator deduction, which California adds back, so taxable income came out ~$147.8k instead of $138,792." +us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It built AGI from gross wages without the $3,088 traditional 401(k) exclusion and applied the full $6,478 capital loss instead of the $3,000 cap. It deducted the educator expense, which California disallows. It also stated that no nonrefundable credits apply, omitting the $1,256 of exemption credits, and understated bracket tax on $138k at ~$4,302 instead of ~$5,870." +us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a fictitious $34,000 standard deduction instead of itemizing $21,274.50 against CA AGI of $160,066.80, and computed $8,100 of tax on $130,894, about $2,860 above the $5,239.62 bracket tax at that income. It applied a renter's credit that is unavailable at this AGI and omitted the $1,256 of exemption credits." us,scenario_099,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -8156,42 +8256,45 @@ us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_out us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model said age 5 is 'within WIC's age limit of under 5', which contradicts itself. A 5-year-old fails the strict under-5 child category test and is ineligible despite the household's low income." us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model read WIC as covering 'children up to age 5' inclusive and counted the 5-year-old as eligible. The child category ends at the 5th birthday, so Child 2 is categorically ineligible." us,scenario_100,child2_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model placed age 5 within WIC's child age range. That range is under 5, so Child 2 fails the categorical test and is ineligible even though income is within limits." -us,scenario_100,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"Its reasoning correctly reached an EITC of $2,366 (40% × $5,915) plus an ACTC of $512 (15% × $3,415), for $2,878. It then submitted $6,425, a number that matches neither its own arithmetic nor any derivation it stated." -us,scenario_100,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It reduced EITC/ACTC earned income to $2,056 by subtracting the $3,859 traditional 401(k) deferral, which gave an EITC of 40% × $2,056 = $822.40 and no ACTC. The reference computes both credits on the full $5,915 gross wages: an EITC of $2,366 plus an ACTC of 15% × $3,415 = $512." -us,scenario_100,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It correctly computed an EITC of $2,366 and an ACTC of 15% × ($5,915 − $2,500) = $512.25. It then replaced the ACTC with an unsupported $834 'approximation'. The refundable CTC is capped at the 15%-of-earnings-over-$2,500 amount, so the total is $2,878, not $3,200." -us,scenario_100,federal_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"It declined to compute the EITC or refundable CTC even though a head of household with two qualifying children and $5,915 of wages is in the EITC phase-in. That phase-in gives 40% × $5,915 = $2,366, and the ACTC is 15% × $3,415 = $512." -us,scenario_100,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It computed an EITC of $2,366 and then cut it to an unexplained $1,906 by invoking a 'smaller of' AGI test. The AGI comparison only matters once income passes the phase-out threshold. In the phase-in range the credit is 40% of earned income, so the EITC stays at $2,366 and the total is $2,878." -us,scenario_100,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It correctly identified an EITC of $2,366 and an ACTC of $512, which sum to $2,878. It then said the EITC was 'reduced slightly' with no rule behind the cut and submitted $2,330." -us,scenario_100,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It added the $710 FLSA overtime premium to wages even though the prompt says gross wages already include overtime, so it counted the overtime twice. It then reported an EITC of $2,905 that matches no stated base. The correct EITC is 40% × $5,915 = $2,366, which with the $512 ACTC gives $2,878." -us,scenario_100,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It used wages net of the $3,859 traditional 401(k) deferral ($2,056) as earned income, which gave an EITC of $822.40 and no ACTC. The reference uses the full $5,915 gross wages for both the EITC phase-in ($2,366) and the ACTC earnings formula ($512)." -us,scenario_100,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It correctly derived an EITC of $2,366 plus an ACTC of $512. It then dropped that result and submitted $6,295 based on unspecified 'IRS EITC tables', which effectively credits a near-maximum EITC to a filer who is still in the 40% phase-in range." -us,scenario_100,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It treated earned income as $6,625 by adding the $710 FLSA overtime premium to the $5,915 gross wages, even though the prompt says gross wages already include overtime. This inflated the EITC to $2,650 and the ACTC to $618.75, instead of $2,366 and $512 on $5,915." -us,scenario_100,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It cut earned income to $928 by subtracting the 401(k) deferral and both $564 health-premium entries as if they were pre-tax payroll deductions, then computed an EITC of $371 and no ACTC. The premiums are not salary reductions in the facts, and the reference applies both credits to the $5,915 gross wages." -us,scenario_100,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It reduced earned income to $2,056 by subtracting the $3,859 traditional 401(k) deferral, which gave an EITC of $822.40 and no ACTC. The reference applies the 40% EITC phase-in and the 15%-over-$2,500 ACTC to the full $5,915 gross wages." -us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It applied the pre-TCJA $3,000 ACTC earnings threshold on the assumption that TCJA expired in 2026, which gave an ACTC of $437.25. The OBBBA made the $2,500 threshold permanent, so the ACTC is 15% × $3,415 = $512 and the total is $2,878." -us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It assumed the household gets the 'full' EITC (roughly the two-child maximum) plus refundable CTC. With $5,915 of earnings, the family is in the phase-in range, so the EITC is only 40% × $5,915 = $2,366 and the ACTC is $512." -us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It treated the $20,644 employer-sponsored insurance premiums as pre-tax deductions that wipe out wages and set earned income to $0. The facts list no salary-reduction arrangement, and the reference computes the EITC ($2,366) and ACTC ($512) on $5,915 of wages." -us,scenario_100,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It assumed pre-tax health insurance premiums reduce earned income to $0 and so denied the EITC and ACTC. The listed ESI premiums do not reduce wages, so the EITC is 40% × $5,915 = $2,366 and the ACTC is $512." -us,scenario_100,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It returned $0 without any computation, missing that a head of household with two qualifying children and $5,915 of wages gets a phase-in EITC of $2,366 and an ACTC of $512." -us,scenario_100,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It computed the ACTC with the expired $3,000 earnings threshold, giving $437.25. For 2026 the threshold is $2,500 (made permanent by the OBBBA), so the ACTC is $512 and the total with the $2,366 EITC is $2,878." -us,scenario_100,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It submitted $0 even though its own explanation concluded $2,803.25, so the submitted value contradicts its reasoning. The $2,803.25 itself relies on the expired $3,000 ACTC threshold instead of the $2,500 threshold that gives a $512 ACTC." -us,scenario_100,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It assumed the TCJA had expired and used a $3,000 ACTC earnings threshold, giving $437.25. The OBBBA kept the threshold at $2,500, so the ACTC is 15% × $3,415 = $512 and the total is $2,878." -us,scenario_100,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It granted the full $1,700-per-child refundable CTC ($3,400) without applying the limit of 15% × (earned income − $2,500) = $512. It also guessed an EITC of $600 from AGI instead of using 40% of the $5,915 earned income ($2,366)." -us,scenario_100,federal_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"It treated the per-child refundable cap as a floor by taking the maximum of $3,300 and the earnings-based $512. The ACTC is the lesser of the per-child cap and 15% × ($5,915 − $2,500), which is $512, so the total is $2,878, not $5,666." -us,scenario_100,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It claimed the income was too low to produce refundable credits. In fact, $5,915 of earnings with two qualifying children produces a phase-in EITC of $2,366 and, because earnings exceed $2,500, an ACTC of $512." -us,scenario_100,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It returned $0 and said no refundable credits are triggered. A head of household with two qualifying children and $5,915 of wages receives a phase-in EITC of $2,366 and an ACTC of $512." -us,scenario_100,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It based the EITC on $2,056 of wages after the $3,859 traditional 401(k) deferral, giving $822.40 and no refundable CTC. The reference uses the $5,915 gross wages, which gives an EITC of $2,366 and an ACTC of $512." -us,scenario_100,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It netted the traditional 401(k) deferral out of earned income ($2,056), which gave an EITC of $822.40 and put earnings below the ACTC threshold. The reference computes both credits on the $5,915 gross wages." -us,scenario_100,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It asserted a zero EITC phase-in amount. A filer with $5,915 of earned income and two qualifying children gets 40% × $5,915 = $2,366 in EITC plus an ACTC of $512." -us,scenario_100,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It took earned income as W-2 wages after the $3,859 401(k) deferral ($2,056), giving an EITC of $822 and no ACTC. The reference applies the EITC and the $2,500-threshold ACTC formula to the full $5,915." -us,scenario_100,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used a $3,000 ACTC earnings threshold, giving $437. For 2026 the threshold is $2,500, so the ACTC is 15% × $3,415 = $512 and the total with the $2,366 EITC is $2,878." -us,scenario_100,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It used W-2 Box 1 wages after the traditional 401(k) deferral ($2,056) as earned income, giving an EITC of $822 and no ACTC. The reference computes the EITC ($2,366) and ACTC ($512) on $5,915 gross wages." -us,scenario_100,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It subtracted the traditional IRA contribution as well as the 401(k) deferral from earned income, leaving $1,876, and got an EITC of $750. An IRA contribution is an above-the-line AGI deduction and never reduces earned income. The reference computes both credits on $5,915, which gives $2,366 plus $512." -us,scenario_100,federal_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"It set earned income to $2,056 of W-2 Box 1 wages after the $3,859 401(k) deferral, giving an EITC of $822.40 and no refundable CTC. The reference uses the $5,915 gross wages, which gives an EITC of $2,366 and an ACTC of $512." -us,scenario_100,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"Its reasoning estimated a large EITC (~$6,500) and an ACTC (~$2,300) but it submitted $0, contradicting its own analysis. The estimates were also wrong: in the phase-in the EITC is 40% × $5,915 = $2,366 and the ACTC is 15% × $3,415 = $512." -us,scenario_100,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It multiplied the earnings-based ACTC amount by the number of children ($512.25 × 2). The limit of 15% × (earned income − $2,500) applies once per family, so the ACTC is $512 and the total is $2,878." -us,scenario_100,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It treated the CTC as refundable up to the full per-child cap ($3,600) without the 15%-of-earnings-over-$2,500 limit, which caps the ACTC at $512. It also inflated the EITC to an arbitrary $3,730 after correctly computing the 40% phase-in amount of $2,366." -us,scenario_100,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It claimed the EITC and refundable CTC are capped by tax liability, but refundable credits are paid regardless of liability. The phase-in EITC of $2,366 and the earnings-based ACTC of $512 are both paid out in full." +us,scenario_100,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the full $5,915 of wages as earned income, but the $3,859 of traditional 401(k) deferrals is excluded from gross income and doesn't count. Correct earned income is $2,055.72, which gives $822.29 EITC and $0 ACTC. It then submitted $6,425, which doesn't even match its own $2,878 calculation." +us,scenario_100,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It computed EITC on $5,915 of gross wages without first removing the $3,859 of excluded 401(k) deferrals. It then added a made-up $834 refundable CTC. Earned income of $2,055.72 is below the $2,500 ACTC floor, so the only credit is EITC of $822.29." +us,scenario_100,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It claimed no refundable credits even though the head has two qualifying children and positive earned income. Earned income of $2,055.72 (wages minus 401(k) deferrals) earns the 40% phase-in EITC of $822.29." +us,scenario_100,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It said pre-tax 401(k) deferrals still count as EITC earned income. They don't: §32(c)(2) counts only compensation includible in gross income, so earned income is $2,055.72. That makes the EITC $822.29 and the ACTC $0, not its $1,906 + $512 mix." +us,scenario_100,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It treated wages as earned income without removing the $3,859 of excluded 401(k) deferrals. Its ad hoc $2,330 combines an EITC and an ACTC that both rest on $5,915. With earned income of $2,055.72, the ACTC is $0 and the EITC is $822.29." +us,scenario_100,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added the $710 FLSA overtime premium on top of gross wages that already include it, and never removed the $3,859 of 401(k) deferrals from earned income. Correct earned income is $2,055.72, which gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It said traditional 401(k) deferrals reduce AGI but not EITC earned income. That's wrong: excluded elective deferrals are not earned income under §32(c)(2), so earned income is $2,055.72. That gives $822.29 EITC and $0 ACTC, since earned income is below $2,500." +us,scenario_100,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It based EITC and ACTC on $5,915 of gross wages instead of $2,055.72 of earned income after the 401(k) exclusion. It then dropped its own $2,878 figure and submitted an invented $6,295 'table' amount." +us,scenario_100,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It deliberately computed EITC on the full $5,915 of wages and missed that the $3,859 of traditional 401(k) deferrals is not earned income. Correct earned income of $2,055.72 gives $822.29 EITC and no ACTC." +us,scenario_100,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It double-counted the $710 overtime premium, which is already inside the $5,915 of wages, to get $6,625. It also never removed the $3,859 of 401(k) deferrals. Correct earned income is $2,055.72, which gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $928 equals wages minus the 401(k) deferrals minus both $564 health insurance premiums. Those premiums are not pre-tax payroll exclusions, so only the $3,859 of 401(k) deferrals comes out. Correct earned income is $2,055.72, giving $822.29 EITC." +us,scenario_100,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used $5,915 of gross wages as earned income and ignored the exclusion of $3,859 of traditional 401(k) deferrals. Correct earned income of $2,055.72 gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It computed EITC on $5,915 instead of $2,055.72 after removing excluded 401(k) deferrals. It also applied a $3,000 ACTC threshold, but the threshold is $2,500. Earned income is under either threshold, so ACTC is $0 and the total is $822.29." +us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It assumed the household gets the full maximum EITC plus refundable CTC. At $2,055.72 of earned income the EITC is still in the 40% phase-in, so it is $822.29. Earned income below $2,500 means no refundable CTC." +us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It treated insurance premiums as pre-tax deductions that reduce earned income to zero. Only the $3,859 of traditional 401(k) deferrals is excluded, leaving $2,055.72 of earned income and an $822.29 phase-in EITC." +us,scenario_100,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It wrongly said pre-tax health insurance premiums reduce earned income to $0. The $20,644 of ESI premiums is employer-paid and not taken out of the listed wages. Only the 401(k) deferrals come out, leaving $2,055.72 of earned income and $822.29 EITC." +us,scenario_100,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It returned $0 with no computation. The head has two qualifying children and $2,055.72 of earned income after excluding 401(k) deferrals, which earns an $822.29 phase-in EITC." +us,scenario_100,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used $5,915 of gross wages as earned income instead of $2,055.72 net of the excluded 401(k) deferrals. It also applied a $3,000 ACTC threshold instead of $2,500. Correct earned income produces $822.29 EITC and no ACTC." +us,scenario_100,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation works out $2,803.25 from $5,915 of gross wages, yet it submitted 0, which contradicts its own reasoning. Both numbers miss the exclusion of the $3,859 of 401(k) deferrals: the correct earned income of $2,055.72 gives $822.29 EITC." +us,scenario_100,federal_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It computed EITC on $5,915 instead of $2,055.72 after the traditional 401(k) exclusion. It also assumed a pre-TCJA $3,000 ACTC threshold, but the threshold stays at $2,500. Earned income is below $2,500, so ACTC is $0 and EITC is $822.29." +us,scenario_100,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It made the refundable CTC $1,700 per child regardless of earnings, ignoring the earned-income formula (15% of earned income over $2,500). With $2,055.72 of earned income that formula yields $0. Its $600 EITC guess should be 40% × $2,055.72 = $822.29." +us,scenario_100,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It invented a $1,650 per-child minimum refundable CTC, which doesn't exist; ACTC is limited to 15% of earned income over $2,500. It also used gross wages instead of the $2,055.72 left after excluding 401(k) deferrals. The correct result is $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It said income is too low for any refundable credit, but the EITC phases in from the first dollar of earned income. $2,055.72 of earned income with two children yields $822.29 at the 40% rate." +us,scenario_100,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It returned $0 and treated low wages as triggering no credits. The EITC phase-in at 40% of $2,055.72 (wages minus 401(k) deferrals) yields $822.29 for a two-child filer." +us,scenario_100,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It took $5,915 of gross wages as earned income for both EITC and ACTC and missed the exclusion of $3,859 of traditional 401(k) deferrals. Correct earned income of $2,055.72 gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It based both credits on $5,915 of gross wages instead of $2,055.72 of earned income net of excluded 401(k) deferrals. That earned income gives $822.29 EITC and is below the $2,500 ACTC floor." +us,scenario_100,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used $5,915 as earned income, missing that the $3,859 of traditional 401(k) deferrals is excluded from gross income and therefore from earned income. Correct earned income of $2,055.72 gives $822.29 EITC and no refundable CTC." +us,scenario_100,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It computed EITC and ACTC on $5,915 of gross wages and never removed the $3,859 of 401(k) deferrals. Correct earned income of $2,055.72 gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It treated $5,915 of gross wages as EITC/ACTC earned income without removing the excluded traditional 401(k) deferrals. Correct earned income is $2,055.72, which gives $822.29 EITC and no ACTC." +us,scenario_100,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It used $5,915 of wages as earned income, but the $3,859 of 401(k) elective deferrals is excluded from gross income and doesn't count. Correct earned income of $2,055.72 gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It wrongly said the phase-in amount is zero. The EITC phase-in is 40% of positive earned income, and $2,055.72 of earned income with two children yields $822.29." +us,scenario_100,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It computed EITC on $5,915 instead of $2,055.72 after the 401(k) exclusion. It also used a $3,000 ACTC threshold instead of $2,500. Correct earned income produces $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It explicitly used gross earned income of $5,915 and missed that traditional 401(k) deferrals are excluded from EITC/ACTC earned income. Correct earned income of $2,055.72 gives $822.29 EITC and no ACTC." +us,scenario_100,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly removed the 401(k) deferrals, but it also subtracted the $180 traditional IRA contribution from earned income. An IRA contribution is an above-the-line deduction and doesn't reduce earned income. Earned income is $2,055.72, so EITC is $822.29, not $750." +us,scenario_100,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used $5,915 of gross wages for both credits instead of $2,055.72 of earned income net of excluded 401(k) deferrals. The correct base gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Its explanation estimates roughly $6,500 EITC and $2,300 ACTC, yet it submitted 0. Both are wrong: earned income after excluding 401(k) deferrals is $2,055.72, so the EITC is in phase-in at $822.29 and ACTC is $0." +us,scenario_100,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the ACTC earned-income formula (15% over $2,500) once per child, but that amount is a household-level limit. It also used gross wages instead of $2,055.72 of earned income after the 401(k) exclusion. Correct earned income gives $822.29 EITC and $0 ACTC." +us,scenario_100,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It treated the CTC as refundable up to $1,800 per child no matter what the household earned, ignoring the 15%-over-$2,500 earned-income limit. It also replaced its own $2,366 EITC with an unsupported $3,730. With $2,055.72 of earned income, the only credit is $822.29 of EITC." +us,scenario_100,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It wrongly said the EITC and refundable CTC are capped by tax liability, but refundable credits don't depend on tax owed. The EITC is 40% of $2,055.72 of earned income (wages minus 401(k) deferrals), or $822.29." us,scenario_100,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted ineligibility with no derivation, never applying either free-tier pathway: SNAP receipt of $8,625.89 categorically confers free meals, and countable income of $11,979.68 (wages $5,914.72 plus TANF $6,064.96) is 44% of the three-person federal poverty guideline, far below the 130% free threshold. Its 0 is consistent with treating SNAP/TANF as absent because receipt is not a listed input fact, even though both are derived outputs of this same household. The income test alone settles it: the $5,915 of wages is about 22% of the poverty guideline, so the household lands in the free tier with two school-age children (ages 6 and 5) generating $7.15 per child per school day, $2,261.92 annually." us,scenario_100,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model gave a circular non-answer — that the benchmark does not indicate positive free-meal support — instead of computing the free-tier tests, so it applied neither SNAP categorical eligibility ($8,625.89 of SNAP automatically confers free meals) nor the 130%-of-poverty income test, which the household passes at 44% FPG on countable income of $11,979.68. Montana has no universal free-meals policy, so the tier turns entirely on those two tests, both of which this household clears. The correct derivation pays $7.15 per school day for each of the two school-age children (ages 6 and 5) for $2,261.92 annually; the model's 0 corresponds to no test being run at all." us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It put 138% FPL for a household of three at about $3,627, when the real figure is about $36,000. That is roughly a tenfold error, and it led the model to conclude that $5,915 exceeded the limit, when $5,915 is only 7% of FPL. It also applied an asset test to the $2,800 in bank assets, but MAGI-category Medicaid (parent/caretaker or expansion adult) has no resource test." @@ -8214,53 +8317,54 @@ us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identi us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed both components correctly ($5,915 × 0.062 = $366.73 and $5,915 × 0.0145 = $85.78, totaling $452.51) and then discarded that result, submitting $477 on the invented rationale of 'rounding to annual benchmark.' No rounding, annualization, or state payroll-tax add-on applies here; the answer its own arithmetic supports is $452.48." us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below 185% of the poverty guideline as sufficient for reduced-price meals and ignored that income at or below 130% qualifies for free meals. At 44% of the guideline, reinforced by categorical eligibility, the household receives the mutually exclusive FREE tier rather than reduced-price support." us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,915 was above 130% of the poverty guideline for a four-person household, even though the engine ratio is 44%. It therefore placed the household in the 130%-185% reduced-price band instead of the FREE tier, while also overlooking categorical eligibility for free meals." -us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Used wages alone ($493 a month) and left out about $505 a month in Montana TANF cash, which counts as unearned income (gross is $998.31). It also missed the $453.64 shelter deduction from the utility allowance and took a $94 medical deduction the head does not qualify for, because the head receives no disability benefit. Its own figure, $758 × 12, is $9,096, not the $9,046 it submitted." -us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted only the $5,915 in wages and left out Montana TANF cash, which counts as unearned income and brings gross to $998.31 a month. It said there were no shelter costs, which missed the $453.64 shelter deduction from the utility allowance. It also took a $1,124-a-year medical deduction, but the head receives no SSI or SSDI and so is not disabled for SNAP purposes." -us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Agreed the household is eligible but never calculated a benefit and submitted an arbitrary 'conservative' $1,908. The correct calculation is $998.31 gross (wages plus TANF), minus $761.21 in deductions, for $237.09 net. That gives $713.90 a month for most of the year and $8,625.89 in total." -us,scenario_100,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Returned $0, citing 'no benefit receipt inferred', even though the prompt says to assume take-up. The household passes every test: gross income is 45% of the poverty line, net income is 11%, and the $2,800 in assets is within the limit. It is owed $713.90–$733.60 a month." -us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Started from wages only and left out Montana TANF cash (about $505 a month), which counts as unearned income. It took a $141 medical deduction that counted the $564 premium twice, even though the head is not disabled for SNAP purposes. It then cut its own $9,108 to $7,320 for a supposed lack of a shelter deduction, but the utility allowance actually produces a $453.64 shelter deduction." -us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Counted wages only and left out about $505 a month in TANF cash. It also did not calculate the $453.64 shelter deduction from the utility allowance and used a low maximum allotment of $766. Its $8,316 is a hand adjustment of an earnings-only $8,496 estimate, not the calculation from $237.09 net income." -us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Treated net income as small because it counted wages alone, leaving out Montana TANF cash, which counts as unearned income and brings gross to $998.31 a month. It also never calculated the $453.64 shelter deduction from the utility allowance. Its $8,300 is a rough guess, not $713.90 × 9 plus $733.60 × 3." -us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Calculated net income of $92 from $493 in wages, leaving out about $505 a month in TANF cash. It took a $94 medical deduction even though the head receives no disability benefit and so is not disabled for SNAP purposes. It also missed the $453.64 shelter deduction from the utility allowance; the correct net income is $237.09." -us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Set shelter costs to $0 because no rent was listed, but the Montana utility allowance produces a $453.64 shelter deduction. It left out TANF cash, which counts as unearned income (gross is $998.31). It also used a $228 standard deduction instead of $209 and took a $93.67 medical deduction the head does not qualify for, ending at a $762.20 monthly benefit instead of $713.90." -us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Counted the $20,644 employer insurance premium as a deductible medical expense to bring net income to zero, and ignored TANF income. It then annualized a $766 monthly benefit as $13,700, more than 12 times any maximum allotment. The correct total is $8,625.89 ($713.90 or $733.60 a month)." -us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Estimated net income at $0–50 by using wages alone and taking a medical deduction for the head. It left out about $505 a month in TANF cash, which counts as unearned income and raises net income to $237.09 even after the $453.64 shelter deduction. The head receives no SSI or SSDI, so there is no medical deduction." -us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium on top of $5,915 in wages, but the prompt says wages already include overtime. It counted the $564 premium twice in a medical deduction the head does not qualify for. It also left out TANF cash and the $453.64 shelter deduction from the utility allowance." -us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income, but at $458.54 instead of about $505.39. Its main error was leaving out the $453.64 shelter deduction from the Montana utility allowance, which left net income at $602.21 instead of $237.09. It also used a $204 standard deduction instead of $209, took a medical deduction the head does not qualify for, and used a $768 maximum instead of $785/$802.60." -us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted TANF at only $335 a month instead of about $505. It left out the $453.64 shelter deduction from the utility allowance and took a $140.67 medical deduction the head does not qualify for, giving net income of $390.98 instead of $237.09. It then submitted $9,037.20, which contradicts its own stated result of $7,808.48." -us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Calculated net income from wages alone, leaving out about $505 a month in TANF cash, which counts as unearned income. It took a $53.67 medical deduction even though the head receives no disability benefit, and missed the $453.64 shelter deduction from the utility allowance. It also used a $204 standard deduction instead of $209." -us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Took a $140 medical deduction that counted the $564 premium twice ($2,108 a year), even though the head is not disabled for SNAP purposes. It ignored TANF cash, which counts as unearned income and brings gross to $998.31. It also left out the $453.64 shelter deduction from the utility allowance, ending at $54 net income instead of $237.09." -us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Its $9,408 ($784 a month) assumes near-zero net income from wages alone. It leaves out about $505 a month in Montana TANF cash, which counts as unearned income. The correct net income is $237.09 and the household contribution is $71.10 a month." -us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Said the household gets 'near the maximum' allotment but submitted $6,128 ($511 a month), which contradicts that claim and matches no calculation. The correct calculation counts TANF as unearned income and deducts $209 standard, $98.58 earned income and $453.64 shelter. That gives $713.90 a month." -us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income, but left out the $453.64 shelter deduction from the Montana utility allowance. It took a medical deduction the head does not qualify for, because the head receives no disability benefit. It also used a $768 maximum instead of $785, giving about $605 a month instead of $713.90." -us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Returned $0, saying the household fails the income and resource limits. Gross income is only 45% of the poverty line, net income is 11%, and the $2,800 in bank assets is within the limit. The household is owed $8,625.89." -us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Started from $492.92 in monthly wages with no TANF cash, so it understated gross income by about $505 a month. It took a medical deduction even though the head receives no SSI or SSDI, and left out the $453.64 shelter deduction from the utility allowance. The result was about $753 a month instead of $713.90." -us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Submitted $9,216 ($768 × 12), which is a full maximum benefit, while its explanation says 9,072. It never counted Montana TANF cash as unearned income, so it missed the $237.09 net income and the $71.10 monthly contribution." -us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Calculated net income of about $97 from wages alone, leaving out about $505 a month in TANF cash, which counts as unearned income. It took a medical deduction the head does not qualify for and left out the $453.64 shelter deduction from the utility allowance. It also used outdated figures: a $204 standard deduction instead of $209 and a $768 maximum instead of $785." -us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Deducted the $20,644 employer insurance premium and other medical costs to bring net income to zero. The head is not disabled for SNAP purposes (no SSI or SSDI), so no medical deduction applies. It also left out TANF cash, which counts as unearned income and keeps net income at $237.09." -us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Calculated net income of $179.34 from wages alone, leaving out about $505 a month in TANF cash. It also left out the $453.64 shelter deduction from the Montana utility allowance. It applied a single $800 maximum for all 12 months instead of $785 for January–September and $802.60 for October–December." -us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Its $12,588 ($1,049 a month) is above the maximum allotment for a three-person household ($785, rising to $802.60). It also ignored the $71.10 monthly contribution that comes from $237.09 net income once TANF cash is counted." -us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Returned $0 because no rent or utility details were given, but SNAP does not require shelter costs. The Montana utility allowance still produces a $453.64 shelter deduction, and the eligible household is owed $8,625.89." -us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Brought net income to zero with medical and premium deductions the head does not qualify for, because the head receives no disability benefit. It ignored TANF cash, which counts as unearned income and leaves net income at $237.09. It also used a $767 maximum instead of $785/$802.60." -us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Assumed zero net income and gave the full $785 × 12. It left out about $505 a month in Montana TANF cash, which counts as unearned income and creates a $71.10 monthly contribution. It also missed the October increase to $802.60." -us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Brought net income to zero with medical deductions for the head, who receives no SSI or SSDI and so gets no medical deduction. It ignored TANF cash, which counts as unearned income and brings net income to $237.09. It then gave the full $785 × 12 instead of $713.90 for most months." -us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI income the head does not receive: the $2,800 in bank assets exceeds the $2,000 SSI resource limit. It also took a medical deduction, but the unearned income is actually Montana TANF cash of about $505 a month. With the $453.64 shelter deduction, the correct benefit is $713.90 a month." -us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Said the head's insurance and medical costs eliminate net income, but the head is not disabled for SNAP purposes (no disability benefit), so no medical deduction applies. It ignored TANF cash, which counts as unearned income and leaves net income at $237.09, so it wrongly gave the full $785 × 12." -us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Said there were no shelter costs, so it left out the $453.64 shelter deduction from the Montana utility allowance. It also left out TANF cash as unearned income. It applied one flat $710.92 for all 12 months, missing the October–December rise to $733.60 under the FY2027 maximum of $802.60." -us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income but subtracted only the earned income and standard deductions. It left out the $453.64 shelter deduction from the Montana utility allowance, so net income was about $557 instead of $237.09 and the benefit $618 a month instead of $713.90." -us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Counted TANF but explicitly applied 'no shelter deduction', missing the $453.64 shelter deduction from the Montana utility allowance. It also took a medical deduction the head does not qualify for, because the head receives no disability benefit. It used a flat $785 maximum for all 12 months, ignoring the October increase." -us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Returned $0 without doing any calculation. The household passes the gross and net income tests and the asset test, with $237.09 net income. That gives $713.90 a month for nine months and $733.60 for three, or $8,625.89." -us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Calculated net income of only $668 a year from wages alone, leaving out about $505 a month in TANF cash. It took a medical deduction that counted the $564 premium twice ($2,108 a year), even though the head is not disabled for SNAP purposes. It also left out the $453.64 shelter deduction and used an outdated $198 standard deduction and $766 maximum." -us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Brought net income to zero with a medical deduction the head does not qualify for, because the head receives no SSI or SSDI. It left out TANF cash, which counts as unearned income and keeps net income at $237.09. It also used a $766 maximum instead of $785/$802.60." -us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Took a $1,688 medical deduction that counted the $564 premium twice, but the head is not disabled for SNAP purposes, so no medical deduction applies. It also ignored TANF cash, which counts as unearned income (about $6,065 a year). It also left out the $453.64-a-month shelter deduction from the Montana utility allowance." -us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Submitted one month's allotment ($805) instead of an annual total. It also brought net income to zero by ignoring TANF cash and taking a medical deduction the head does not qualify for. The correct benefit is $713.90–$733.60 a month, or $8,625.89 a year." -us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"Estimated net income near $50 using wages alone plus a medical deduction for the head, who receives no disability benefit and so gets no medical deduction. It left out about $505 a month in TANF cash, which counts as unearned income, and never calculated the $453.64 shelter deduction. The correct net income is $237.09." -us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was nothing to score against the $8,625.89 annual benefit." -us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Brought net income to zero with premium and medical deductions the head does not qualify for, because the head receives no SSI or SSDI. It ignored TANF cash, which counts as unearned income and leaves net income at $237.09 and a $71.10 monthly contribution. It also missed the October increase to $802.60." -us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted all of its $393 net income from a $766 maximum instead of 30% of it, ending at $377 a month. It also skipped the $209 standard deduction, the $453.64 shelter deduction and TANF income." -us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Took a $182 medical deduction based on $2,608 a year in costs, which included the child's $500 bill and the premium twice. The head is not disabled for SNAP purposes, so no medical deduction applies at all. It also left out TANF cash, which counts as unearned income, so net income was $3 instead of $237.09." -us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium to wages even though the prompt says wages already include it. It then turned a $766 monthly figure into $12,444 a year, more than the $9,192 that 12 months at $766 would give. It also ignored TANF income and the $71.10 monthly contribution." -us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Said the household's resources exceed SNAP limits, but the $2,800 in bank assets is within the limit, and the household is categorically eligible through TANF. It is owed $8,625.89." +us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"The model counted only the $493/month in wages and left out the roughly $505/month of non-wage benefit income that brings SNAP gross income to $998.31. It also subtracted a $94 medical deduction, giving $89 net income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction. It then applied $785 to all 12 months and miscomputed $758×12 as $9,046." +us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model built net income from wages alone ($5,915/yr) and missed the roughly $505/month of unearned benefit income in SNAP gross income. It took a $1,124/yr medical deduction and no shelter deduction, giving $86.67/month net and $9,108. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction." +us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model concluded the household was eligible but never computed a benefit, and gave an arbitrary 'conservative' $1,908. The record's r30_snap_heat_and_eat_sua correction gives $691 net, a rounded $208 contribution and $577/month ($6,924/year), using the FY2026 $785 maximum held all year. The excluded frozen reference's $8,625.89 includes the defect-based shelter deduction and projected FY2027 October–December parameters." +us,scenario_100,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"The model returned $0 even though the household passes every test: gross income is 45% of FPG, the $2,800 in assets is within limits, and TANF non-cash categorical eligibility applies. The record's r30_snap_heat_and_eat_sua correction gives $691 net income and $577/month; about $714/month is the frozen reference's January–September amount, which includes the defect-based shelter deduction." +us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model computed net income from wages alone, double-counted the $564 premium in a $141 medical deduction, and got a near-zero net income ($29) and $9,108. It then arbitrarily cut that to $7,320. It never added the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction." +us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model's net income of about $194/month counts wages only and leaves out the roughly $505/month of non-wage benefit income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month. It used a $766 maximum allotment instead of the FY2026 $785 held for all 12 months; $802.60 is the frozen engine's October–December projection. It then made an unexplained cut from $8,496 to $8,316." +us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"The model assumed net income was just wages less the earned income and standard deductions, leaving out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net, a $208 contribution and $577/month. Its $8,300 is below its own wages-only formula; holding the FY2026 $785 maximum for all 12 months matches c_snap_hold_fy2026, whereas the frozen reference's $71.10 contribution and October–December $802.60 maximum reflect the shelter defect and engine projection." +us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model's net income of $92 uses $493 of wages only, plus a $94 medical deduction that PolicyEngine does not apply. It left out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 contribution rather than its approximately $27.60; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction." +us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"The model treated wages as the only income and subtracted a $93.67 medical deduction, getting $72.67 net. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; $237.09 is the frozen reference's defect-based net. It also used a $228 standard deduction instead of $209 and a flat $784 maximum for all 12 months." +us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model counted the $20,644 employer-sponsored premium as a deductible out-of-pocket medical expense to drive net income to zero, and it left out the roughly $505/month of non-wage income. Its $13,700 is even higher than its own $766×12 = $9,192 maximum, which is an arithmetic error." +us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model assumed net income of $0–50/month from wages alone plus a medical deduction. It missed the roughly $505/month of non-wage benefit income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based $453.64 shelter deduction." +us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model added the $710 overtime premium on top of $5,915 in wages, even though overtime is already included in gross wages. It double-counted the $564 premium in a medical deduction, left out the roughly $505/month of non-wage income, and used a $797 maximum, giving $120 net income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net; $237.09 is the frozen reference's defect-based net." +us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model included estimated TANF income ($458.54/month) with wages and obtained $602.21 net income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction. It also used a $768 maximum allotment instead of $785." +us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model counted only $335/month of TANF instead of the roughly $505/month of non-wage income and used a $198 standard deduction instead of $209. It added a $140.67 medical deduction (double-counting the premium) and used a $768 maximum. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect. Its explanation's $7,808.48 also contradicts the $9,037.20 it submitted." +us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,The model's $136.67 net income uses wages only with a $204 standard deduction and a $53.67 medical deduction. It left out the roughly $505/month of non-wage income and used an $805 maximum instead of the FY2026 $785 held for all 12 months. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect. +us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model counted only wages and subtracted a $140 medical deduction that double-counts the $564 premium, getting $54 net. It missed the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based $453.64 shelter deduction." +us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,The model assumed a near-maximum benefit (~$784/month) without counting the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net income and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction. +us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model said the household was near the maximum allotment but submitted $6,128 (~$511/month), well below the $785 maximum, with no supporting calculation. The record's r30_snap_heat_and_eat_sua correction gives $691 net, a rounded $208 contribution and $577/month, with the FY2026 $785 maximum held all year. The frozen reference's approximately $714 and $733.60 monthly amounts include the shelter defect and projected FY2027 October–December parameters." +us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model included TANF as unearned income and took no utility-allowance shelter deduction, as the record's r30_snap_heat_and_eat_sua correction does. That correction gives $691 net and $577/month; $237.09 is the frozen reference's net, which includes the defect-based $453.64 shelter deduction. It also used a $768 maximum instead of $785, giving $7,257." +us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"The model claimed the household fails income and resource limits. Gross income is only 45% of FPG, the $2,800 in assets is within limits, and TANF non-cash categorical eligibility applies, so SNAP is positive under the record's stated-facts correction: $577/month after correcting r30_snap_heat_and_eat_sua; about $714/month is the frozen reference's January–September amount, which includes the defect-based shelter deduction." +us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model based net income on $492.92 of wages alone with a medical deduction. It left out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $6,924/year; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction, and the model's $9,036 is above both annual comparisons." +us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model gave a near-maximum benefit with no income calculation and left out the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 contribution; $237.09 is the frozen reference's net, which includes the defect-based shelter deduction. Its submitted $9,216 also contradicts the $9,072 in its explanation." +us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"The model calculated net income of $97 from wages only, using a $204 standard deduction and a medical deduction. It left out the roughly $505/month of non-wage income and used a $768 maximum instead of the FY2026 $785 held for all 12 months. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction and $802.60 October–December maximum come from the defect and engine projection." +us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"The model treated the $20,644 employer-sponsored premium as a deductible out-of-pocket medical cost to zero out net income, and ignored the roughly $505/month of non-wage income. It also used an $800 maximum for all 12 months, whereas the $785 maximum applies through September." +us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,The model's $179.34 net income uses wages only and omits non-wage benefit income (~$505/month). Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect. It used an $800 maximum for every month instead of the FY2026 $785 held all year under c_snap_hold_fy2026; $802.60 is the frozen engine's October–December projection. +us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model's $12,588 works out to $1,049/month, which is above the 3-person maximum allotment of $785. It also never subtracted the $71.10 contribution from $237.09 of net income." +us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model set SNAP to zero because no rent was listed. SNAP does not require shelter costs, and the household is eligible through low income and TANF non-cash categorical eligibility, with $691 net income and $577/month under the record's r30_snap_heat_and_eat_sua correction; $237.09 net and about $714/month are the frozen reference's January–September figures, which include the defect-based shelter deduction." +us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"The model assumed medical and premium expenses wiped out countable income and paid the full $767/month. PolicyEngine counts about $505/month of non-wage income and applies no medical deduction, so net income is $237.09 and the contribution is $71.10." +us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,The model paid the full $785×12 maximum on the assumption of zero net income. It left out the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction. +us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model used medical premiums and expenses to zero out net income and paid the $785×12 maximum. PolicyEngine applies no medical deduction and counts about $505/month of non-wage income, leaving $237.09 net and a $71.10 monthly contribution." +us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model counted benefit income as SSI, although the record has SSI at $0 and roughly $505/month of TANF. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month. Its approximately $533/month is below that comparison; the frozen reference's approximately $714/month includes the defect-based shelter deduction." +us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,The model claimed insurance and medical expenses eliminate net income and paid the $785×12 maximum. It left out the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction. +us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model computed a single $710.92 monthly amount from wages alone with no shelter deduction and multiplied it by 12. Holding the FY2026 schedule all year matches c_snap_hold_fy2026, and taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction. With roughly $505/month of TANF counted, that correction gives $691 net, a $208 contribution and $577/month. The frozen reference's $237.09 net, $71.10 contribution and October–December $733.60 benefit on a $802.60 maximum reflect the shelter defect and engine projection." +us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model included estimated TANF cash income and applied only the earned income and standard deductions, as the record's r30_snap_heat_and_eat_sua correction does. That correction gives $691 net, a $208 contribution and $577/month, below its $618/month; the frozen reference's $237.09 net and approximately $714/month include the defect-based $453.64 shelter deduction." +us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model explicitly set the shelter deduction to zero while counting TANF income, as the record's r30_snap_heat_and_eat_sua correction does. That correction gives $691 net, a $208 contribution and $577/month, below its approximately $647/month; the frozen reference's $713.90 ($785 − $71.10) includes the defect-based $453.64 shelter deduction." +us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"The model returned $0 without any calculation. The household is income-, asset- and categorically eligible under the record's stated-facts reading, whose r30_snap_heat_and_eat_sua correction gives $691 net and $577/month; $237.09 net and about $714/month are the frozen reference's January–September figures, which include the defect-based shelter deduction." +us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,The model's $668/year net income counts wages only and uses a $198 standard deduction plus a medical deduction that double-counts the $564 premium. It left out the roughly $505/month of non-wage income and used a $766 maximum instead of $785. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect. +us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"The model asserted net income of $0 and paid $766×12. It ignored the roughly $505/month of non-wage income and used $766 instead of $785. The record's r30_snap_heat_and_eat_sua correction subtracts the $209 standard and $98.58 earned income deductions but no utility-allowance shelter deduction, giving $691 net and a $208 contribution; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction." +us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"The model built net income ($536/year) from wages alone with a $1,688 medical deduction that counts the $564 premium twice. It left out the roughly $505/month of non-wage income and used a $787/month maximum. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect." +us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model reported $805, a single monthly allotment, instead of the annual total. It also assumed zero net income, ignoring the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction." +us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"The model's net income of about $50/month uses $493 of wages and a medical deduction. It left out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net and about $714/month include the defect-based $453.64 shelter deduction." +us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so there was no usable answer to compare with the excluded frozen reference's $8,625.89 annual value." +us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"The model assumed premiums and medical expenses zero out net income and paid the $785×12 maximum. PolicyEngine counts about $505/month of non-wage income and applies no medical deduction, so net income is $237.09 and the contribution is $71.10/month." +us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"The model's $393 net income subtracts only the earned income deduction, not the standard deduction. It then subtracted roughly the full net income from a $766 maximum instead of 30% of it, giving about $377/month. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a rounded $208 contribution, so its monthly figure is $785 − $208 = $577; about $714 is the frozen reference's January–September figure, which includes the defect-based shelter deduction." +us,scenario_100,snap,muse-wonder,prompt_ambiguity,age_disability,False,"The model's $97 net income is wages less the earned income, standard, and medical deductions. It left out the roughly $505/month of non-wage income, giving a contribution of about $29. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 contribution; $71.10 is the frozen reference's contribution, which includes the defect-based $453.64 shelter deduction." +us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"The model claimed a $182/month medical deduction from $2,608/yr of costs. That double-counts the $564 premium and includes Child 2's $500, though Child 2 is not elderly or disabled. Combined with leaving out the roughly $505/month of non-wage income, this drove net income to $3. The record's r30_snap_heat_and_eat_sua correction gives $691 net and $577/month; $237.09 is the frozen reference's net, which includes the defect-based shelter deduction." +us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model added the $710 overtime premium on top of wages that already include it. Its $12,444 is above its own $766×12 = $9,192 maximum, an arithmetic error, and it never computed the $71.10 contribution from $237.09 net income." +us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model claimed resources exceed SNAP limits. The $2,800 bank balance is within the limit and the household is categorically eligible through TANF non-cash status, so SNAP is positive under the record's stated-facts correction: $577/month after correcting r30_snap_heat_and_eat_sua; about $714/month is the frozen reference's January–September amount, which includes the defect-based shelter deduction." us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head's generic disability fact as satisfying SSI's categorical disability requirement and proceeded directly to the earned-income calculation. PolicyEngine's SSI-specific aged/blind/disabled indicator is false for the head, so no SSI benefit-rate or income-exclusion calculation applies; its submitted value also contradicts its own stated $9,480 calculation." us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated the head's generic disability status with SSI categorical eligibility. The head's SSI-specific aged/blind/disabled indicator is false, so the earned-income reduction it applied is irrelevant and SSI is zero." us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the head was an SSI-qualified disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails SSI's aged, blind, or qualifying-disabled criterion, so the calculation stops before income exclusions or the federal benefit rate are applied." @@ -8270,138 +8374,141 @@ us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." us,scenario_100,ssi,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed the 46-year-old head's generic disability status met SSI's qualifying-disability test and then calculated countable wages. The SSI-specific categorical indicator is false, so the head is ineligible regardless of the wage calculation; neither child qualifies either." us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the generic disability input as SSI categorical eligibility and therefore performed an income-reduction calculation that should never occur. It also added the separately listed overtime premium to annual gross wages and invented an IRWE adjustment despite instructions not to infer unlisted expenses, but categorical ineligibility alone makes SSI zero." -us,scenario_100,state_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"The model used a 10% Montana EITC rate instead of the 2026 rate of 20%. It also guessed earned income at about $2,646 after a 401(k) adjustment. The federal EITC is 40% of the full $5,915 in wages, or $2,365.89, so the Montana credit is $473.18; the model's confused math produced $265." -us,scenario_100,state_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"The model subtracted the $3,859 traditional 401(k) deferral from wages, giving a $822.40 federal EITC (40% × $2,056). The correct federal EITC is $2,365.89 on the full $5,915. The model then applied a 10% Montana rate instead of the 2026 rate of 20%." -us,scenario_100,state_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"The model said Montana has no refundable earned income credit. Montana's EITC is refundable and pays 20% of the federal EITC in 2026, which is $473.18 on a $2,365.89 federal EITC. The model also wrongly used being below the filing threshold as a reason for $0, but a refundable credit is paid even with no tax liability." -us,scenario_100,state_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"The model gave no derivation and left out the Montana refundable EITC completely. The correct amount is 20% of the $2,365.89 federal EITC (40% of $5,915 with two qualifying children), or $473.18." -us,scenario_100,state_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"The model computed the federal EITC correctly at about $2,366, then applied a 10% Montana rate instead of the 2026 rate of 20%. It also turned its own $237 result into $297 with no basis. The correct credit is 20% × $2,365.89 = $473.18." -us,scenario_100,state_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"The model said a positive federal EITC and a Montana state EITC both applied, then reported $0 anyway as a 'conservative' estimate. The refundable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18, and is paid even with no tax liability." -us,scenario_100,state_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"The model treated the Montana EITC as nonrefundable, so zero tax liability wiped it out. The Montana EITC is refundable, and at 20% of the $2,365.89 federal EITC it pays $473.18 regardless of tax owed." -us,scenario_100,state_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"The model based the federal EITC on wages minus the traditional 401(k) deferral, giving $822.40, instead of the full $5,915, which gives $2,365.89. It also applied 10% instead of Montana's 2026 rate of 20%." -us,scenario_100,state_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"The model correctly found a refundable Montana EITC on a federal EITC of about $2,366, but used a 10% rate. Montana's 2026 rate is 20%, so the credit is $473.18, not $237." -us,scenario_100,state_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"The model said the Montana EITC is nonrefundable and set refundable credits to $0. The Montana EITC is refundable and equals 20% of the $2,365.89 federal EITC, which is $473.18." -us,scenario_100,state_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"The model used the right federal EITC base (about $2,366) but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18, not $236.60." -us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"The model used a 12% Montana EITC rate that does not exist instead of the 2026 rate of 20%. It also overstated the federal EITC at $2,650; the correct figure is 40% × $5,915 = $2,365.89. The correct Montana credit is $473.18." -us,scenario_100,state_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"The model put the federal EITC at $371, as if most of it had phased out, but at $5,915 of earnings with two children the credit is still in the 40% phase-in range and equals $2,365.89. The model also used a 10% Montana rate instead of 20%. The correct Montana credit is $473.18." -us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"The model used the outdated 3% Montana EITC rate on a federal EITC that had been reduced by the 401(k) deferral to $822.40. The correct figure is 20% of $2,365.89. It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits, and the value it submitted ($24.67) does not match its stated total ($1,224.67)." -us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"The model said Montana has no refundable credits for this household and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18, not $236.60." -us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"The model claimed the household has zero earned income and that Montana has no refundable EITC. Both claims are wrong: the head earns $5,915, which produces a $2,365.89 federal EITC, and Montana's refundable EITC pays 20% of that, or $473.18." -us,scenario_100,state_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"The model set the federal EITC to $0, which also zeroed out the Montana EITC. A head of household with $5,915 of earnings and two qualifying children (ages 6 and 5) gets 40% × $5,915 = $2,365.89, so the Montana credit is $473.18." -us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"The model gave no derivation and left out the Montana refundable EITC. The correct amount is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"The model's reasoning used the outdated 3% Montana EITC rate to get $70.98, yet it submitted $0, which contradicts its own explanation. The 2026 rate is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"The model claimed PolicyEngine does not model Montana refundable credits. The Montana EITC is modeled and refundable at 20% of the $2,365.89 federal EITC, which gives $473.18." -us,scenario_100,state_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state income tax. Montana does levy one, and it includes a refundable EITC equal to 20% of the federal EITC, or $473.18 here." -us,scenario_100,state_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but the 3% Montana rate from 2024. The 2026 rate is 20%, which gives $473.18." -us,scenario_100,state_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"The model found no Montana refundable credits and missed the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"The model assumed a low-income household gets no Montana refundable credit. The refundable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"The model put the federal EITC at $4,207, which is more than 40% × $5,915 = $2,365.89 allows in the phase-in range. It also used a 10% Montana rate instead of 20%. The correct Montana credit is $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"The model used a 10% Montana EITC rate ($236.60) instead of 20% ($473.18). It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits; the EITC alone is the total." -us,scenario_100,state_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"The model used a 10% Montana EITC rate ($236.60) instead of 20% ($473.18). It also added a $1,200 young-child credit for the five-year-old, but this household's Montana refundable credits consist only of the EITC." -us,scenario_100,state_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"The model used the correct 20% Montana rate but a federal EITC of $822.40. That figure is 40% of wages after subtracting the $3,859 traditional 401(k) deferral. The engine uses the full $5,915 of earned income, giving a $2,365.89 federal EITC and a $473.18 Montana credit." -us,scenario_100,state_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"The model got the Montana EITC right at $473.20 (20% of the federal EITC). It then added $1,800 of Montana child tax credit, but this household's Montana refundable credits are the EITC alone." -us,scenario_100,state_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"The model used the correct 20% rate but on an $822.40 federal EITC, which came from cutting earned income by the traditional 401(k) deferral. With the full $5,915 of earned income, the federal EITC is $2,365.89 and the Montana credit is $473.18." -us,scenario_100,state_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state EITC. Montana has a refundable EITC that is 20% of the federal EITC in 2026, or $473.18 here." -us,scenario_100,state_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state EITC and only checked property-tax-style credits. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"The model ruled out the elderly homeowner/renter credit correctly but wrongly said Montana has no state EITC. The refundable Montana EITC is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and left out the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"The model put the federal EITC at about $750 instead of $2,365.89 and used a 10% Montana rate instead of 20%. It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits. The correct total is the EITC alone, $473.18." -us,scenario_100,state_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no refundable state EITC. Montana's refundable EITC equals 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,state_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18." -us,scenario_100,state_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no state income tax. Montana does levy one, and it includes a refundable EITC equal to 20% of the federal EITC, or $473.18 here." -us,scenario_100,state_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"The model used the right $2,366 federal EITC but a 10% Montana rate. The 2026 rate is 20%, so the credit is $473.18." -us,scenario_100,state_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"The model claimed Montana has no earned income tax credit. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, or $473.18, even with no Montana tax liability." -us,scenario_100,state_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"The model said no Montana refundable credits apply and missed the refundable Montana EITC. That credit is 20% of the $2,365.89 federal EITC, or $473.18." -us,scenario_100,tanf,claude-fable-5,llm_error,categorical_eligibility,False,"Its own reasoning found the family eligible, with countable income of about $220 a month and a positive benefit of about $368 a month, but it then submitted $0. It also used a $588–$633 payment standard instead of Montana's 2026 three-person standard of about $725 a month, which leaves a benefit of about $505 a month." -us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It wrongly said Montana TANF is not modeled. The program is modeled, and this family qualifies: the about $725 payment standard minus $219.69 in countable earnings (after the $200 + 25% disregard) gives about $505 a month, or $6,064.96 a year." -us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It counted a parent and two children as a household of four instead of three. It also used a $2,000 asset limit, but Montana TANF's resource limit is $3,000, so the $2,800 bank balance qualifies. Countable earnings of $219.69 a month are far below the payment standard." -us,scenario_100,tanf,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It returned $0 because the facts did not show TANF receipt, but the prompt says to assume program take-up. TANF should have been computed from eligibility: about $725 minus $219.69 in countable earnings each month, or $6,064.96 a year." -us,scenario_100,tanf,claude-opus-4.7,llm_error,categorical_eligibility,False,"It defaulted to $0 by vaguely pointing to asset, work, and income tests without applying any of them. Assets are under Montana's $3,000 limit with the vehicle excluded, 55 hours a week meets the work rules, and countable earnings of $219.69 leave a benefit of about $505 a month." -us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"It claimed a $595 monthly benefit, which is above the $588 maximum it cited itself, so it never subtracted countable earnings. The correct benefit is the about $725.10 payment standard minus $219.69 in countable earnings, or about $505.41 a month ($6,064.96 a year)." -us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It assumed PolicyEngine gives this family no Montana TANF. The family meets the resource test and has countable earnings of only $219.69 a month after the $200 + 25% disregard, which leaves about $505 a month under the about $725 payment standard." -us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"Its eligibility logic and the $200 + 25% earnings disregard ($220 a month countable) were correct, but it guessed a $730 payment standard. The 2026 three-person standard averages about $725.10 a month, so the benefit is $505.41 a month, not $510." -us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It used a $1,000 liquid-asset limit, but Montana TANF's resource limit is $3,000, so $2,800 in bank assets passes. The family is eligible and gets about $505 a month after the earnings disregard." -us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $327 monthly payment standard, which is far below Montana's 2026 three-person standard of about $725. It also paid that near-maximum amount without subtracting the $219.69 in countable earnings left after the $200 + 25% disregard." -us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled and returned $0. The program is modeled, and this family qualifies for about $725 minus $219.69 in countable earnings each month, or $6,064.96 a year." -us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to bank assets in the resource test. Montana TANF excludes the household vehicle, so countable resources are $2,800, under the $3,000 limit, and the family is eligible." -us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"It disregarded 50% of earnings above $200, but Montana disregards only 25%, so countable earnings are $219.69, not $146.46. It also used a $605 payment standard instead of the 2026 three-person standard of about $725.10." -us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its reasoning computed a positive benefit of $4,023.75 but it submitted $0. That calculation also used a $555 payment standard instead of about $725.10, which understates the correct benefit of about $505 a month." -us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $500 payment standard and a 50% disregard of earnings above $200. Montana disregards only 25%, so countable earnings are $219.69, and the 2026 three-person standard is about $725.10, which gives about $505 a month." -us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the $200 + 25% disregard correctly ($220 countable) but used an outdated $588 payment standard. The 2026 three-person standard is about $725.10, so the benefit is about $505 a month, not $368." -us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It claimed the family fails Montana's TANF requirements without naming any test. The family passes the $3,000 resource test, and its $219.69 in countable earnings is well below the about $725 payment standard, so it is owed about $505 a month." -us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,asset_resource,False,"It assumed the head receives SSI, but the head's $2,800 in resources is over the $2,000 SSI limit for an individual, so there is no SSI income to offset TANF. With wages as the only income, TANF is about $505 a month." -us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $5,298 total implies a 50% disregard of earnings above $200 and an outdated $588 payment standard. Montana disregards only 25%, so countable earnings are $219.69, and the 2026 standard is about $725.10." -us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave $0 with no supporting derivation. The family is eligible, and the benefit is the about $725.10 payment standard minus $219.69 in countable earnings, or $6,064.96 a year." -us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"$5,298.48 equals ($588 − $146.46) × 12, so it used an outdated $588 payment standard and a 50% disregard. Montana's disregard is $200 plus 25% (countable $219.69), and the 2026 three-person standard is about $725.10." -us,scenario_100,tanf,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Its explanation says the family qualifies and states a value of 4,824, yet it submitted $0. Even the $4,824 figure ($402 a month) understates the correct benefit of about $725.10 minus $219.69, or about $505 a month." -us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It wrongly said Montana TANF is not modeled in PolicyEngine. The program is modeled, and it pays this family $6,064.96 a year based on the three-person payment standard minus disregarded earnings." -us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $710 overtime premium on top of $5,915 in wages, even though gross wages already include overtime. It also compared gross income to the payment standard without the $200 + 25% disregard. Countable earnings are only $219.69 a month, assets pass with the vehicle excluded, and 55 hours a week meets the work rules." -us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,"It assumed Montana TANF gives this household nothing. The family is eligible, and the about $725.10 payment standard minus $219.69 in countable earnings gives $6,064.96 a year." -us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It required an explicit indication of TANF payability, ignoring the instruction to assume take-up. A low-earning single parent with two children and assets under $3,000 qualifies for about $505 a month." -us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It missed that a parent with two dependent children, low earnings, and resources under Montana's $3,000 limit meets TANF eligibility. The benefit is about $725.10 minus $219.69 each month, or $6,064.96 a year." -us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"Its $150 countable income and $438 grant imply a 50% disregard and a $588 payment standard. Montana counts 75% of earnings above $200 ($219.69), and the 2026 three-person standard is about $725.10, so the benefit is about $505 a month." -us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $359.50 monthly benefit implies a payment standard below $590. The 2026 Montana three-person standard is about $725.10, and subtracting $219.69 in countable earnings gives about $505 a month." -us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"$4,419.75 is ($588 − $219.69) × 12, so it applied the disregard correctly but used an outdated $588 payment standard. The 2026 three-person standard PolicyEngine applies is about $725.10." -us,scenario_100,tanf,gpt-5.6-terra,llm_error,asset_resource,False,"It counted SSI income the head does not receive: the head's $2,800 in resources is over the $2,000 SSI limit, so SSI is $0. Countable earnings of $219.69 a month alone leave a TANF benefit of about $505 a month." -us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"Its method was correct ($200 + 25% disregard, $2,636.25 a year in countable earnings), but it used a rounded $725 monthly payment standard. The 2026 standard PolicyEngine applies averages about $725.10 a month, which leaves its total $1.21 short of $6,064.96." -us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"$5,298 matches a $588 payment standard with a 50% disregard of earnings above $200. Montana counts 75% of the excess ($219.69 a month), and the 2026 three-person standard is about $725.10." -us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"Its $368.25 monthly benefit comes from an outdated $588 payment standard minus $219.69 in countable earnings. The 2026 three-person standard is about $725.10, which gives about $505 a month." -us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,It applied the $200 + 25% disregard and the resource test correctly but used a $588 monthly payment standard instead of the 2026 three-person standard of about $725.10. That understates the benefit by about $137 a month. -us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"It did not compute TANF at all. The family is eligible, and the benefit is the about $725.10 payment standard minus $219.69 in countable earnings, or $6,064.96 a year." -us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"Its disregard ($220 countable) and asset findings were correct, but it used an outdated $588 payment standard. The 2026 three-person standard is about $725.10, which gives about $505 a month." -us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"It subtracted $220 in countable earnings from an outdated $588 payment standard. The 2026 Montana three-person standard is about $725.10, so the monthly benefit is about $505, not $368." -us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"Every step was correct except the payment standard: it used $588 instead of the 2026 three-person standard of about $725.10. That understates the annual benefit by about $1,645." -us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the disregard correctly but used a $675 payment standard. The 2026 three-person standard is about $725.10, so the benefit is about $505 a month, not $455." -us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"It took the $588 maximum grant as the 2026 payment standard. PolicyEngine's 2026 three-person standard is about $725.10, and subtracting $220 in countable earnings gives about $505 a month." -us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no TANF value and no explanation, so there was no answer to score against $6,064.96." -us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"Its resource test and $219.69 countable-earnings calculation were correct, but it used an outdated $588 three-person grant instead of the 2026 payment standard of about $725.10." -us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"It assumed Montana benefits are only $50–$200 a month and that earnings reduce them to zero. The three-person payment standard is about $725.10, and countable earnings after the $200 + 25% disregard are just $219.69, which leaves about $505 a month." -us,scenario_100,tanf,ox-alpha,llm_error,asset_resource,False,"It counted about $780 a month in SSI as unearned income, but the head is not SSI-eligible because $2,800 in resources is over the $2,000 SSI limit. With only $219.69 in countable earnings, TANF is about $505 a month." -us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $710 overtime premium on top of $5,915 in wages, even though gross wages already include overtime. It then compared gross income to an annual limit without the $200 + 25% earnings disregard, which brings countable earnings down to $219.69 a month." -us,scenario_100,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"It said income was too high without applying Montana's $200 + 25% earnings disregard. Countable earnings are $219.69 a month, well below the about $725.10 payment standard, so the benefit is $6,064.96 a year." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model subtracted $1,400 of auto loan interest on top of the $16,100 standard deduction, which lowered taxable income to $83,630. The facts never show a new, U.S.-assembled vehicle bought with a post-2024 loan, so no qualified passenger vehicle loan interest deduction applies. Its final $12,064 does not even match its own $13,058–$13,111 working, and the correct figure is $13,418.60 on $85,030." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model's standard deduction and 2026 brackets were right, but it also subtracted a $1,000 phased-down auto loan interest deduction. The listed loan is not shown to be a qualified passenger vehicle loan, so that deduction does not apply. That left taxable income at $84,030 instead of $85,030 and understated tax by $220." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model counted the nontaxable $700 of veterans benefits in AGI. It used the 2024 $14,600 standard deduction and $11,600 bracket edge instead of the 2026 figures ($16,100 and $12,400/$50,400), and it taxed nearly all income at 12% while skipping the 22% bracket. Its stated $10,637.50 does not match its submitted $11,435." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model subtracted the health premiums from wages as if they were pre-tax, but no pre-tax exclusion is listed. It showed no bracket arithmetic. Its $13,900 implies about $87,200 of taxable income under 2026 brackets, roughly $2,200 above the correct $85,030, which fits an old standard deduction of about $15,000 rather than $16,100." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed exactly $13,418.60 from $85,030 of taxable income and the $12,400/$50,400 bracket edges. It then threw that result away and submitted $14,716 under the label 'rounding to bracket estimates', a number with no basis in its own working." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model derived taxable income of $85,030 and tax of about $13,419 using the correct 2026 brackets. It then replaced that with an unsupported $13,876, which none of its stated parameters produce." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model took the full $1,400 of auto loan interest as a qualified vehicle loan interest deduction, dropping taxable income to $83,630. Nothing establishes a new, U.S.-assembled vehicle with a post-2024 loan, so the only deduction is the $16,100 standard deduction. Tax should be on $85,030." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 OBBBA standard deduction of $15,750 instead of the 2026 indexed $16,100. It also used guessed bracket edges of $12,300/$49,950 instead of $12,400/$50,400. This overstated taxable income by $350 and misplaced the bracket boundaries, giving $13,543." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model got taxable income right at $85,030 but used wrong 2026 bracket edges ($12,150/$49,400 instead of $12,400/$50,400), which gave $13,523.60. It then submitted an unexplained $13,322 that matches neither its own working nor the correct $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted a $1,174 auto loan interest deduction, lowering taxable income to $83,856. The loan is not shown to be a qualified passenger vehicle loan (new vehicle, U.S. final assembly, originated after 2024), so no such deduction applies. Taxable income stays at $85,030." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $15,600 standard deduction instead of the 2026 $16,100, and bracket edges of $12,175/$49,550 instead of $12,400/$50,400. Taxable income came out at $85,530, and pushing more income into the 22% bracket inflated tax to $13,618." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted $3,840 of 'pre-tax' health premiums from wages, counting the same $1,920 premium twice, even though no salary-reduction exclusion is listed. It also used a $15,400 standard deduction and $12,200/$49,700 bracket edges instead of $16,100 and $12,400/$50,400." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset rules: an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA rate structure permanent and set the 2026 single standard deduction at $16,100 with no personal exemption, so taxable income is $85,030, taxed at 10/12/22%." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $15,350 standard deduction instead of $16,100, and bracket edges of $12,200/$49,575 instead of $12,400/$50,400. Taxable income came out at $85,780 and tax at $13,670.10 instead of $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired, using an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. Under OBBBA, 2026 keeps 10/12/22% brackets with a $16,100 standard deduction and no exemption. It also subtracted $1,920 of premiums as pre-tax with no listed exclusion." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model started from $103,000 of wages and ignored the $1,870 capital loss that reduces AGI to $101,130. It used a $15,300 standard deduction instead of $16,100, ending with $87,700 of taxable income instead of $85,030." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model got AGI right at $101,130 but showed no deduction or bracket calculation. Its $15,400 is about $1,980 too high: subtracting the $16,100 standard deduction and applying 10% to $12,400, 12% to $50,400 and 22% above gives $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated 2026 as a post-TCJA-expiration year, using an $8,250 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA brackets permanent, and the 2026 single standard deduction is $16,100 with no exemption." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used a standard deduction of about $15,000 and never worked through the brackets. Its $11,102 implies only about $74,500 of taxable income under 2026 brackets, roughly $10,500 short of the correct $85,030 (AGI $101,130 minus the $16,100 standard deduction)." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset rules (an $8,300 standard deduction plus a $5,300 personal exemption, with 15%/25% brackets) and also used outdated $11,000/$44,700 bracket edges. Under current law, 2026 uses 10/12/22% brackets with edges at $12,400/$50,400 and a $16,100 standard deduction." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model subtracted an $8,300 standard deduction and a $5,300 personal exemption on the assumption that the TCJA sunset in 2026. OBBBA kept the TCJA structure, so the correct deduction is $16,100 with no exemption, and $85,030 is taxed at 10/12/22%." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket edges of $11,925/$48,475. The 2026 figures are a $16,100 deduction and $12,400/$50,400 edges, which give $85,030 of taxable income and $13,418.60 of tax." -us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model extrapolated its own 2026 parameters: a $15,450 standard deduction and bracket edges of $12,283/$49,929. The published 2026 values are $16,100 and $12,400/$50,400, so its taxable income was $650 too high and its brackets were in the wrong places." -us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model's 2026 brackets were right, but it used a $15,800 standard deduction instead of the 2026 indexed $16,100. That left taxable income at $85,330 instead of $85,030 and overstated tax by $66." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model added 'taxable pension income' and 'small taxable capital gains' to wages. The $2,308 pension is tax-exempt, and the $1,870 net capital loss reduces AGI to $101,130 rather than adding gain. That inflated taxable income above $85,030 and tax above $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model claimed itemized medical effects, but $2,520 of medical costs is far below the 7.5%-of-AGI floor ($7,585), and the standard deduction wins anyway. It showed no bracket calculation. Its $9,587 is about $3,800 below the $13,418.60 that results from $85,030 of taxable income under 2026 brackets." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model assumed the scheduled TCJA sunset and used a pre-TCJA standard deduction plus a personal exemption, leaving about $87,380 of taxable income taxed at the old higher rates. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent for 2026 with no exemption." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model subtracted a phased-down auto loan interest deduction on top of the standard deduction. The listed loan is never shown to be a qualified passenger vehicle loan (new, U.S.-assembled vehicle, loan originated after 2024), so that deduction is zero. Taxable income stays at $85,030 rather than $84,030." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model subtracted a $1,000 auto loan interest deduction to reach $84,030 of taxable income. No qualified passenger vehicle loan is established, so the only deduction is the $16,100 standard deduction, giving $85,030 and $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction instead of $16,100, which overstated taxable income at $85,830. Its $13,774 also exceeds what the correct 2026 brackets produce on that income ($13,594.60), so it used bracket edges below $12,400/$50,400 as well." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset structure: about $8,400 of standard deduction plus a $5,300 exemption, taxed at 10/15/25%. Current law for 2026 keeps 10/12/22% brackets and a $16,100 standard deduction with no exemption." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket edges of $11,925/$48,475 instead of the 2026 values ($16,100 and $12,400/$50,400). That overstated taxable income by $1,100 and misplaced the bracket boundaries." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model got taxable income right at $85,030 but used bracket edges of $12,200/$49,575 instead of the 2026 values of $12,400/$50,400. That pushed $825 too much income into the 22% bracket and $200 too much into the 12% bracket, overstating tax by $86.50." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA reverted in 2026, using an $8,300 standard deduction, a $5,300 exemption, and 15%/25% brackets. OBBBA made the TCJA rates permanent, with a $16,100 standard deduction and no exemption for 2026." -us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly reached $85,030 of taxable income, but it reported an approximate tax of $13,426 instead of summing the brackets exactly. The exact sum is $1,240 + $4,560 + 22% × $34,630 = $13,418.60, so its figure is $7.40 high." -us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this variable, so no answer could be scored." -us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted a $1,174 auto loan interest deduction, cutting taxable income to $83,856. The loan is not established as a qualified passenger vehicle loan, so no such deduction applies. Taxable income stays at $85,030." -us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model's own reasoning acknowledged that the standard deduction and 2026 brackets leave a positive tax, but it submitted $0 anyway. With no credits to offset it, tax on $85,030 of taxable income is $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $15,750 standard deduction instead of $16,100 and bracket edges of $12,150/$49,000 instead of $12,400/$50,400, which gave $13,641. It then submitted $14,827, which contradicts its own calculation." -us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model netted the $1,870 capital loss against tax-exempt pension income instead of deducting it from AGI, and it used a pre-TCJA-style $13,600 deduction instead of $16,100. It taxed everything above $12,400 at 12%, skipping the 22% bracket. It then subtracted employee payroll tax as an invented nonrefundable credit." +us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 10% Montana match rate instead of the 2026 rate of 20%. It switched between earned-income figures of $2,056 and $2,646, and its final $265 doesn't follow from any of its own figures. The correct path is 20% × (40% × $2,055.72 = $822.29) = $164.46." +us,scenario_100,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model got the federal EITC about right at roughly $822 by excluding the 401(k) deferral. However, it applied the outdated 10% Montana match instead of the 2026 rate of 20%, which cut the credit in half to $82.24 instead of $164.46." +us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed Montana has no income-based refundable credit and pointed to the household being below the filing threshold. But Montana's refundable EITC equals 20% of the federal EITC and pays out with no tax liability, so this filer gets 20% × $822.29 = $164.46." +us,scenario_100,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model gave no refundable credit and offered no reasoning. It missed the refundable Montana EITC, which is 20% of the $822.29 federal EITC that a head of household with two children and $2,055.72 of earned income receives, or $164.46." +us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a 10% Montana match instead of 20%. It also figured the federal EITC on $5,915 of gross wages without removing the $3,859 traditional 401(k) deferral, getting $2,366 instead of $822.29. Then it reported $297, which doesn't match even its own $237 result." +us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model recognized that the federal EITC is positive and that Montana provides a state EITC, but it still entered zero as a 'conservative' estimate. Montana's EITC is refundable, and 20% of the $822.29 federal EITC is $164.46." +us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model treated the Montana EITC as nonrefundable and zeroed it out because the household has no tax liability. The credit is refundable and pays 20% of the $822.29 federal EITC, or $164.46." +us,scenario_100,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model correctly started from a federal EITC of about $822 that excludes the 401(k) deferral. It then applied a 10% Montana match instead of the 2026 rate of 20%, getting $82.24 instead of $164.46." +us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a 10% Montana match instead of 20%. It also figured the federal EITC as 40% of $5,915 in gross wages, even though it noted the $3,859 traditional 401(k) deferral and should have used the remaining $2,055.72 of taxable wages, which gives $822.29." +us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said the Montana EITC is nonrefundable and reported zero. The credit is refundable at 20% of the federal EITC, which gives $164.46 on the $822.29 federal credit." +us,scenario_100,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model applied the outdated 10% Montana match to a $2,366 federal EITC. That federal figure is 40% of gross wages, with no exclusion for the $3,859 traditional 401(k) deferral. The correct figure is 20% × $822.29 = $164.46." +us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a 12% Montana match instead of 20%. It also used a $2,650 federal EITC instead of $822.29, which is 40% of the $2,055.72 left after the 401(k) deferral." +us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 10% Montana match instead of 20%, and it understated the federal EITC at $371. The federal credit is 40% × $2,055.72 = $822.29, so the Montana credit is $164.46." +us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied the old 3% Montana EITC rate to an $822.40 federal EITC and submitted $24.67. The 2026 rate is 20%, which gives $164.46. Its explanation also added a $1,200 child credit that isn't part of the computed Montana refundable credits for this household." +us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model asserted that no Montana refundable credit applies. It missed the refundable Montana EITC of 20% of the $822.29 federal EITC, which is $164.46." +us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 10% match instead of 20%, and it used a $2,366 federal EITC, which is 40% of gross wages with the $3,859 traditional 401(k) deferral left in. The federal credit should be $822.29." +us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model reported no state refundable credits. It overlooked Montana's refundable EITC, which is 20% of the $822.29 federal EITC, or $164.46." +us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model claimed Montana has no refundable EITC and that the filer has zero earned income. The filer has $2,055.72 of earned income after the 401(k) deferral, which produces an $822.29 federal EITC and a refundable Montana EITC of 20% of that, or $164.46." +us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model set the federal EITC to $0. A head of household with two qualifying children and $2,055.72 of earned income is in the 40% phase-in range and gets $822.29. That carries through to a $164.46 Montana EITC." +us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,The model gave zero without reasoning. It missed the refundable Montana EITC of 20% × $822.29 = $164.46. +us,scenario_100,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used the correct 20% Montana rate but based it on a $2,366 federal EITC, which is 40% of gross wages. It didn't exclude the $3,859 traditional 401(k) deferral from earned income, which leaves $2,055.72 and an $822.29 federal credit." +us,scenario_100,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's explanation applied the old 3% Montana rate to a $2,366 federal EITC that leaves the 401(k) deferral in earned income, giving $70.98. It then submitted $0, which contradicts its own explanation. The correct figure is 20% × $822.29 = $164.46." +us,scenario_100,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model claimed PolicyEngine doesn't model Montana refundable credits. PolicyEngine computes a refundable Montana EITC of 20% of the federal EITC, which is $164.46 here." +us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model wrongly said Montana has no individual income tax. Montana does tax income, and it provides a refundable EITC of 20% of the $822.29 federal EITC, or $164.46." +us,scenario_100,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used the old 3% Montana EITC rate instead of 20%. It also used a $2,366 federal EITC that leaves the $3,859 traditional 401(k) deferral in earned income, instead of $822.29." +us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model found no refundable Montana credits. It missed the refundable Montana EITC of 20% × $822.29 = $164.46 that this low-earning parent of two receives. +us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed low income meant no refundable Montana credit. Low positive earnings with two children is exactly the case that produces a federal EITC, and Montana refunds 20% of it, or $164.46." +us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a 10% match instead of 20%, and it badly overstated the federal EITC at $4,207. The federal credit is only 40% of $2,055.72 in earned income after the 401(k) deferral, or $822.29." +us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a 10% match instead of 20%. It also used a $2,366 federal EITC computed on gross wages, when the $3,859 traditional 401(k) deferral should have been excluded from earned income to give $822.29." +us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,other,False,"The model added a $1,200 young-child credit that isn't part of the computed Montana refundable credits for this household. Its EITC also used the outdated 10% rate on a $2,366 federal EITC that leaves the 401(k) deferral in. The only refundable credit is the EITC: 20% × $822.29 = $164.46." +us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model added a $1,200 young-child credit that isn't among the computed Montana refundable credits for this household. Its EITC also used 10% of a $2,366 federal EITC, instead of 20% of the $822.29 federal EITC that excludes the 401(k) deferral." +us,scenario_100,state_refundable_credits,gpt-6-luna,llm_error,other,False,"The model added an $1,800 Montana child tax credit that isn't part of the computed refundable credits. It also applied the correct 20% rate to a $2,366 federal EITC that didn't exclude the $3,859 traditional 401(k) deferral, instead of to $822.29." +us,scenario_100,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model used the correct 20% Montana rate but applied it to a $2,366 federal EITC figured on gross wages. The $3,859 traditional 401(k) deferral comes out of EITC earned income, so the federal credit is $822.29 and the Montana credit is $164.46." +us,scenario_100,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model asserted that no Montana refundable credits apply. It missed the refundable Montana EITC of 20% × $822.29 = $164.46. +us,scenario_100,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model claimed Montana has no state EITC. Montana provides a refundable EITC of 20% of the federal EITC, which is $164.46 here." +us,scenario_100,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Montana has no state EITC and only checked elderly and property-tax credits. Montana's refundable EITC of 20% of the $822.29 federal EITC applies, giving $164.46." +us,scenario_100,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model ruled out the elderly homeowner/renter credit, then wrongly said Montana has no state EITC. The refundable Montana EITC is 20% × $822.29 = $164.46." +us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model said no Montana refundable credits apply. It missed the refundable Montana EITC of 20% × $822.29 = $164.46. +us,scenario_100,state_refundable_credits,inkling,llm_error,other,False,"The model added a $1,200 refundable child credit that isn't part of the computed Montana refundable credits. It also applied a 10% rate to a roughly $750 federal EITC instead of 20% of $822.29." +us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model claimed Montana offers no refundable state EITC. Montana's EITC is refundable at 20% of the federal EITC, giving $164.46 for this filer." +us,scenario_100,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 10% match instead of 20%, and it used a $2,366 federal EITC based on gross wages. That figure leaves the $3,859 traditional 401(k) deferral in earned income instead of the correct $822.29." +us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model wrongly said Montana has no state income tax. Montana has an income tax and a refundable EITC of 20% of the federal EITC, which is $164.46 here." +us,scenario_100,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"The model treated Montana's earned income credit as nonrefundable and reported zero. The credit is refundable at 20% of the $822.29 federal EITC, or $164.46." +us,scenario_100,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied a 10% match instead of 20%, and it used a $2,366 federal EITC figured on gross wages. It didn't remove the $3,859 traditional 401(k) deferral, which leaves $2,055.72 of earned income and an $822.29 federal credit." +us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model claimed Montana has no earned income tax credit. Montana provides a refundable EITC of 20% of the federal EITC, which is $164.46 here." +us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,The model said no Montana refundable credits apply. It missed the refundable Montana EITC of 20% × $822.29 = $164.46. +us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly found the family eligible with about $220 of countable income, then reversed itself and submitted $0, claiming earnings exceed the standard. Its own numbers ($588–$633 standard minus $220) give a positive benefit. The correct $725 standard less $219.69 gives about $505/month." +us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,The model assumed Montana TANF is not modeled and returned $0. PolicyEngine does model it: about $725/month less $219.69 of countable earnings is roughly $505/month for this eligible family of three. +us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"The model used a $2,000 asset limit and a household of 4 (there are 3 people), and compared gross wages to the income limit without the disregard. Montana's resource limit is $3,000, so $2,800 passes. After the $200 + 25% disregard, countable income of $219.69 is far below the ~$725 payment standard." +us,scenario_100,tanf,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model returned $0 because no TANF receipt was listed, but the prompt says to assume program take-up. The family qualifies for about $725 − $219.69 ≈ $505/month." +us,scenario_100,tanf,claude-opus-4.7,llm_error,asset_resource,False,"The model returned $0, saying asset and vehicle limits were met and no application was indicated. Take-up is assumed, $2,800 is under the $3,000 limit, the vehicle is excluded, and $219.69 of countable earnings leaves about $505/month under the ~$725 standard." +us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"The model's $595/month is roughly a full maximum grant based on the old ~$588 standard; it never subtracted the $219.69 of countable earnings. The correct figure is the ~$725 standard minus $219.69, about $505/month." +us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"The model assumed PolicyEngine gives no Montana TANF for this family. Countable earnings of $219.69/month (after the $200 + 25% disregard) are well under the ~$725 standard, so the benefit is about $505/month." +us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"The model applied the disregard correctly ($219.69 countable) but rounded the payment standard to $730 instead of PolicyEngine's ~$725. That gives $510/month ($6,120/year) instead of $505.41/month ($6,064.96/year)." +us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"The model used a $1,000 liquid-asset limit and found $2,800 disqualifying. Montana's TANF resource limit is $3,000, so the family passes and gets about $725 − $219.69 ≈ $505/month." +us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a made-up $327/month maximum grant and a 20%-style disregard. PolicyEngine's three-person standard is about $725, and subtracting $219.69 of countable earnings (after the $200 + 25% disregard) gives about $505/month, not $327." +us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,The model assumed Montana TANF is not modeled and returned $0. The engine pays about $505/month: the ~$725 standard minus $219.69 of countable earnings. +us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"The model counted the $1,981 vehicle toward resources to reach $4,781. The vehicle is excluded, leaving $2,800, which is under the $3,000 limit, so the family gets about $505/month." +us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $605 standard and disregarded 50% of earnings above $200 ($146.46 countable). Montana disregards only 25% of the remainder, so countable earnings are $219.69, and the standard is about $725, giving about $505/month." +us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model computed a positive benefit ($4,023.75) but submitted $0. Its computation also used a $555 standard instead of PolicyEngine's ~$725; with $219.69 countable, the benefit is about $505/month." +us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $500 standard and a 50% disregard of earnings above $200. The correct inputs are a ~$725 standard and a 25% disregard ($219.69 countable), which give about $505/month." +us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the $200 + 25% disregard correctly but used an outdated $588 three-person standard. PolicyEngine's 2026 standard is about $725, so the benefit is about $505/month, not $368." +us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said the household fails Montana's requirements without naming a test. It passes all of them: $2,800 of assets is under $3,000, and $219.69 of countable earnings is under the ~$725 standard, giving about $505/month." +us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,asset_resource,False,"The model assumed the head gets SSI and that SSI blocks TANF. SSI is $0 because $2,800 of bank assets exceeds the $2,000 SSI resource limit, so the family gets the full TANF calculation of about $505/month." +us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model's $5,298 equals $588 − $146.46: an outdated $588 standard plus a 50% disregard. Montana disregards 25% of earnings above $200 ($219.69 countable) against a ~$725 standard, which gives about $505/month." +us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave $0 with no reasoning. The family is eligible, and the ~$725 standard minus $219.69 of countable earnings gives about $505/month ($6,064.96/year)." +us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $5,298.48 equals ($588 − $146.46) × 12: an outdated $588 standard and a 50% disregard. The correct calculation is the ~$725 standard minus $219.69, using the 25% disregard." +us,scenario_100,tanf,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's explanation says the family qualifies for $4,824, but it submitted $0. Even $4,824 ($402/month) is short of the correct ~$725 − $219.69 ≈ $505/month." +us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model claimed Montana TANF is not modeled in PolicyEngine. It is modeled, and this family gets about $505/month (~$725 standard less $219.69 of countable earnings)." +us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model added the $710 overtime premium on top of wages that already include overtime, then compared unreduced income to the standard without the $200 + 25% disregard. Countable earnings are $219.69/month, well under the ~$725 standard, and no work-hours test removes the benefit." +us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,The model assumed PolicyEngine pays no Montana TANF here. The engine gives about $505/month: the ~$725 standard minus $219.69 of countable earnings. +us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model returned $0 because TANF receipt wasn't indicated, ignoring the instruction to assume take-up. The eligible family gets about $725 − $219.69 ≈ $505/month." +us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said there were no TANF triggers, but a parent with two children under 18, low earnings, and $2,800 of assets meets Montana's categorical, income, and resource tests. With take-up assumed, the benefit is about $505/month." +us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated $150 of countable income, which reflects a 50% disregard, and a $588 standard. Montana disregards only 25% of earnings above $200, giving $219.69 countable, and the 2026 standard is about $725, so the benefit is about $505/month." +us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $359.50/month comes from a too-low three-person standard (around $588). PolicyEngine's 2026 standard is about $725, and minus $219.69 of countable earnings that gives about $505/month." +us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model's $4,419.75 is ($588 − $219.69) × 12: the disregard is right but the standard is an outdated $588. PolicyEngine uses about $725, giving $6,064.96/year." +us,scenario_100,tanf,gpt-5.6-terra,llm_error,asset_resource,False,"The model counted SSI income that does not exist: the head's $2,800 of bank assets exceeds the $2,000 SSI resource limit, so SSI is $0. With only $219.69 of countable earnings against the ~$725 standard, TANF is about $505/month." +us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"The method is right (disregard to $219.69 countable, subtracted from the three-person standard), but the model used a rounded $725 standard. PolicyEngine's uprated 2026 standard is slightly higher (≈$725.10), so the annual total came out $1.21 short of $6,064.96." +us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $5,298 matches an outdated $588 standard with a 50% disregard ($146.46 countable). The correct inputs are a 25% disregard ($219.69 countable) and a ~$725 standard, giving about $505/month." +us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"The model's $368.25/month comes from an outdated $588 standard. PolicyEngine's 2026 three-person standard is about $725, and minus $219.69 that gives about $505/month." +us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model applied the $200 + 25% disregard and the asset test correctly but used an outdated $588 standard. The ~$725 2026 standard is what gives $6,064.96/year." +us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"The model did no TANF calculation and defaulted to $0. The family is eligible, and the ~$725 standard minus $219.69 of countable earnings gives about $505/month." +us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"The model used an outdated $588 three-person standard. With the correct ~$725 standard and the same $219.69 countable income, the benefit is about $505/month, not $368." +us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"The model used an outdated $588 standard. PolicyEngine's 2026 standard is about $725, so subtracting $219.69 of countable income gives about $505/month." +us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"The disregard, resource, and work logic are correct, but the model used an outdated $588 standard instead of the ~$725 2026 standard. That understates the benefit by about $137/month." +us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,The model used a $675 three-person maximum instead of PolicyEngine's ~$725 standard. That gives $455/month instead of about $505.41/month. +us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"The model used an outdated $588 maximum grant. With the ~$725 2026 standard and $219.69 of countable earnings, the benefit is about $505/month ($6,064.96/year)." +us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"No TANF value or explanation was returned, so nothing could be parsed for this output." +us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"The resource test and $219.69 of countable earnings are correct, but the model used an outdated $588 standard. PolicyEngine's ~$725 standard gives about $505/month." +us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"The model assumed Montana TANF pays only $50–$200/month and that earnings wipe it out. The three-person standard is about $725, and countable earnings after the $200 + 25% disregard are only $219.69, leaving about $505/month." +us,scenario_100,tanf,muse-wonder,llm_error,thresholds_rates,False,The model's $368/month comes from an outdated $588 standard. PolicyEngine's ~$725 2026 standard minus $219.69 gives about $505/month. +us,scenario_100,tanf,ox-alpha,llm_error,asset_resource,False,"The model counted about $780/month of SSI as unearned income. SSI is $0 because the head's $2,800 of bank assets exceeds the $2,000 SSI resource limit, so the only countable income is $219.69 of earnings against the ~$725 standard." +us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model added the $710 overtime premium to wages that already include it, and compared gross annual income to the limit without the $200 + 25% disregard. Countable income is $219.69/month, well below the ~$725 standard." +us,scenario_100,tanf,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model said income was too high without applying the disregard. $492.92/month of wages reduces to $219.69 of countable income, well under the ~$725 three-person standard, so the benefit is about $505/month." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,400 auto loan interest as a qualified vehicle loan deduction. The household facts never say the loan is for a new, U.S.-assembled vehicle, so no deduction applies and taxable income is $85,030. It then submitted $12,064, which matches neither of its own computed figures ($13,111 or $13,058)." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its brackets and standard deduction were correct, but it subtracted a $1,000 auto loan interest deduction. The loan is not shown to be a qualifying new, U.S.-assembled vehicle loan, so the deduction is $0. It also applied the phaseout to the interest itself rather than to the $10,000 cap. Removing the deduction gives $85,030 of taxable income and $13,418.60 in tax." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted the $700 of nontaxable veterans benefits in AGI and used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It also skipped the 22% bracket entirely. Its submitted $11,435 does not even match its own $10,637.50 calculation." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted health premiums from wages as if they were pre-tax, but wages are given as gross and nothing marks the premiums as pre-tax. It also used approximate brackets instead of the 2026 thresholds. The correct path is $85,030 of taxable income and $13,418.60 in tax, while it submitted $13,900." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It correctly computed taxable income of $85,030 and tax of $13,418.60. It then submitted $14,716 with an unexplained 'rounding to bracket estimates' adjustment that contradicts its own correct arithmetic." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It reached $85,030 of taxable income and tax of about $13,419 using the correct 2026 brackets. It then replaced that figure with an unexplained $13,876." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the full $1,400 auto loan interest as a qualified vehicle loan interest deduction. That deduction requires a loan for a new, U.S.-assembled vehicle, which is not among the facts, so it is $0. Its taxable income of $83,630 is therefore $1,400 too low, which understates the tax by $308." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $15,750 (the 2025 OBBBA standard deduction) instead of the 2026 amount of $16,100. It also invented 2026 bracket thresholds of $12,300 and $49,950 instead of $12,400 and $50,400, which produced $13,543." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the right $85,030 of taxable income but used wrong bracket thresholds ($12,150 and $49,400 instead of $12,400 and $50,400), giving $13,523.60. It then submitted an unrelated $13,322." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted about $1,174 of auto loan interest as a qualified vehicle loan deduction. The loan is not shown to be for a new, U.S.-assembled vehicle, so the deduction is $0. That cut taxable income from $85,030 to $83,856 and understated the tax at $13,160." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,600 standard deduction instead of $16,100, and bracket thresholds of $12,175 and $49,550 instead of $12,400 and $50,400. That inflated taxable income to $85,530 and the tax to $13,618." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $3,840 of health premiums from wages, which double-counts the same $1,920 premium and treats it as a pre-tax exclusion the facts never establish. It also used a $15,400 standard deduction and $12,200/$49,700 brackets instead of $16,100 and $12,400/$50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law with an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent, so the correct inputs are a $16,100 standard deduction, no exemption and 10/12/22% brackets." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of $16,100, and bracket thresholds of $12,200 and $49,575 instead of $12,400 and $50,400. That gave taxable income of $85,780 and tax of $13,670.10." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA made the TCJA permanent, so the 2026 standard deduction is $16,100 under 10/12/22% brackets. It also subtracted $1,920 of premiums as pre-tax without any basis in the facts." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left out the $1,870 capital loss, which reduces AGI to $101,130. It also used a $15,300 standard deduction instead of $16,100, which inflated taxable income to $87,700 instead of $85,030." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its AGI of $101,130 was correct, but it never applied the 2026 $16,100 standard deduction and brackets ($12,400/$50,400) explicitly. Its $15,400 guess overstates the correct $13,418.60 by about $1,981." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a post-TCJA-expiration regime with an $8,250 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the TCJA permanent, so the correct inputs are a $16,100 standard deduction and 10/12/22% brackets." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no bracket computation and assumed a standard deduction of about $15,000. Correctly applying the $16,100 deduction to $101,130 of AGI and the 2026 brackets gives $13,418.60, so its $11,102 understates the tax by about $2,317." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied expired-TCJA law: an $8,300 standard deduction, a $5,300 exemption, and 10/15/25% brackets with outdated $11,000/$44,700 thresholds. OBBBA kept the $16,100 standard deduction and 10/12/22% brackets." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used the TCJA-sunset $8,300 standard deduction plus a $5,300 personal exemption, producing $87,530 of taxable income at pre-TCJA rates. Under OBBBA the deduction is $16,100 with no exemption, which gives $85,030 at 10/12/22%." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 bracket thresholds of $11,925 and $48,475. The 2026 values are $16,100 and $12,400/$50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the standard deduction at $15,450 instead of $16,100, and the bracket thresholds at about $12,283 and $49,929 instead of $12,400 and $50,400. That overstated the tax at $13,611.04." +us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its brackets ($12,400 and $50,400) were correct, but it used a $15,800 standard deduction instead of the 2026 amount of $16,100. That leaves $300 too much taxable income and overstates the tax by $66, at $13,484.60." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated the tax-exempt private pension as taxable income and described the capital loss as net taxable gains. AGI is just $103,000 minus $1,870, or $101,130, which gives $85,030 of taxable income after the $16,100 standard deduction. Its $14,498 overstates the tax." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed an itemized medical-expense benefit, but the $2,520 of premiums and medical costs falls well below the 7.5%-of-AGI floor of $7,585, so the standard deduction applies. Its $9,587 is far below the $13,418.60 that $85,030 of taxable income produces." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It assumed the TCJA would sunset and claimed a standard deduction plus a personal exemption, giving about $87,380 of taxable income. OBBBA made permanent the $16,100 standard deduction with no exemption and the 10/12/22% brackets." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted a phased-down $1,000 qualified auto loan interest deduction. The household facts never establish a new, U.S.-assembled vehicle loan, so the deduction is $0 and taxable income stays at $85,030. The extra deduction understated the tax by $220." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It took a $1,000 auto loan interest deduction after the phaseout. The loan is not shown to be for a qualifying new, U.S.-assembled vehicle, so no deduction applies. That reduced taxable income to $84,030 instead of $85,030." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 amount of $16,100, giving $85,830 of taxable income. Its projected brackets were also off, which produced $13,774." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA would sunset and applied an $8,400 standard deduction, a $5,300 exemption and 10/15/25% brackets. OBBBA kept the $16,100 standard deduction and 10/12/22% brackets for 2026." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 $15,000 standard deduction and the 2025 bracket thresholds of $11,925 and $48,475. The 2026 OBBBA values are $16,100 and $12,400/$50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $85,030 of taxable income was correct, but it used bracket thresholds of $12,200 and $49,575 instead of the 2026 values of $12,400 and $50,400. That overstated the tax by $86.50." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied reverted pre-TCJA law with an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. OBBBA made the TCJA permanent, so the 2026 standard deduction is $16,100 under 10/12/22% brackets." +us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its $85,030 of taxable income was correct, but its bracket arithmetic gave $13,426 instead of $1,240 + $4,560 + 22% × $34,630 = $13,418.60. That means it used thresholds slightly different from the 2026 values of $12,400 and $50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there was no answer to score." +us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,174 auto loan interest deduction. The loan is not shown to be for a new, U.S.-assembled vehicle, so the deduction is $0. It also applied the 20% phaseout to the interest rather than to the $10,000 cap. Without the deduction, taxable income is $85,030 and tax is $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It acknowledged that the 2026 brackets leave a positive tax, then submitted $0 anyway. With $85,030 of taxable income and no nonrefundable credits, the tax is $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Its $85,030 of taxable income was correct, but it reported $13,487 instead of $1,240 + $4,560 + $7,618.60 = $13,418.60. That means it used wrong 2026 bracket thresholds instead of $12,400 and $50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction instead of $16,100, and $12,150/$49,000 thresholds instead of $12,400/$50,400, which gave $13,641. It then submitted an unrelated $14,827." +us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It wrongly netted the capital loss against the tax-exempt pension instead of deducting it from wages, and used a $13,600 deduction instead of $16,100. It applied 12% to all income above $12,400 with no 22% bracket. It then subtracted employee payroll tax as if it were a nonrefundable credit, which no rule allows." us,scenario_101,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It correctly computed $6,386 Social Security plus $1,493.50 Medicare = $7,879.50. It then submitted $8,149.50, citing a made-up 'adjustment for wage base cap and rounding.' The cap does not bind at $103,000 of wages, so no adjustment applies and the submitted value contradicts its own arithmetic." us,scenario_101,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its components ($6,386 Social Security plus about $1,494 Medicare) add to $7,879.50, but it submitted $8,803. That figure does not follow from any of the components it listed, so this is an addition error. No extra payroll tax applies in Texas." @@ -8586,106 +8693,109 @@ us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correc us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value and no reasoning, so there was no answer to score. The correct derivation is 7.65% employee FICA on $7,656 of wages, which gives $585.68." us,scenario_108,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_108,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test to its ~$2,217 net income and declared the household ineligible. It missed that Wisconsin BBCE (TANF non-cash categorical eligibility, 200% FPL gross limit) waives the net income test. A categorically eligible one-person household receives the $24/month minimum allotment ($288/yr)." -us,scenario_108,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Ruled the household ineligible because ~$2,214 net income exceeds the ~$1,305 net limit. It ignored Wisconsin BBCE: gross income of $2,581.83 is under 200% FPL, so the household is categorically eligible and the net income test is waived. Its zero formula benefit therefore becomes the $24/month one-person minimum allotment." -us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Tested countable income against a 130% FPL limit, which does not govern here. Wisconsin BBCE uses a 200% FPL gross limit that $2,581.83/month passes, and elderly/disabled households are exempt from the 130% test anyway. The household is categorically eligible and gets the $24/month minimum allotment." -us,scenario_108,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Asserted a zero benefit without any computation and never applied the minimum-allotment rule. The household is categorically eligible through Wisconsin BBCE, so even though its formula benefit is zero, it receives the $24/month one-person minimum ($288/yr)." -us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"Its own reasoning reached the minimum-benefit outcome, then it discarded that and submitted an unsupported $2,227. The correct result is the minimum allotment for a categorically eligible one-person household: 30% of net income ($611) exceeds the $298 maximum. The FY2026 minimum is $24/month ($288/yr), not the $23 it cited." -us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Its reasoning concluded the household fails the 100% FPL net income test, but it submitted an unrelated $2,376. Both steps are wrong: Wisconsin BBCE waives the net income test, and a categorically eligible one-person household whose formula benefit is negative gets only the $24/month minimum ($288/yr)." -us,scenario_108,snap,claude-opus-5,llm_error,thresholds_rates,False,"It reached the minimum-allotment outcome but used the outdated $23 minimum. The FY2026 one-person minimum is 8% of the $298 maximum, rounded to $24/month, which gives $288/yr instead of its $276. Its reasoning also overstated the medical deduction (~$465 instead of ~$32/month)." -us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Explicitly asserted that Wisconsin has no broad-based categorical eligibility and then failed the household on the net income test. Wisconsin FoodShare does use BBCE at 200% FPL gross, which $2,581.83/month passes. That waives the net income test, and the household receives the $24/month minimum allotment." -us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Assumed net income of about $1,700 and a $164/month regular benefit, an unsupported excess shelter deduction. Actual net income is about $2,036, so the 30% contribution of $611 exceeds the $298 maximum and the formula benefit is zero. Only the $24/month categorical-eligibility minimum applies ($288/yr)." -us,scenario_108,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test and declared the household ineligible. Wisconsin BBCE (gross income under 200% FPL) makes the household categorically eligible, which waives the net income test and guarantees the $24/month one-person minimum allotment." -us,scenario_108,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Failed the household on the 100% FPL net income test using net income of ~$2,184. It missed Wisconsin's TANF non-cash BBCE pathway, which waives that test, and the rule that categorically eligible one-person households get the $24/month minimum allotment." -us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the maximum allotment, but treated the negative result as $0. Under Wisconsin BBCE the household is categorically eligible, so a one-person household receives the $24/month minimum allotment ($288/yr) instead of zero." -us,scenario_108,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Found that the 30% contribution exceeds the one-person maximum and stopped at zero. It never applied the minimum-benefit rule: categorically eligible one- and two-person households receive $24/month in FY2026, which gives $288/yr." -us,scenario_108,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test to an elderly/disabled household. Wisconsin BBCE confers categorical eligibility that waives this test, and the household receives the $24/month one-person minimum allotment." -us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Claimed gross income exceeds SNAP limits and zeroed the benefit because 30% of net income exceeds the maximum. Gross income of $2,581.83 is under Wisconsin's 200% FPL BBCE limit, so the household is categorically eligible and receives the $24/month minimum allotment." -us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Asserted that income exceeds the eligibility threshold, but $2,581.83/month passes Wisconsin's 200% FPL BBCE gross limit. Through that categorical eligibility, the one-person household receives the $24/month minimum allotment ($288/yr)." -us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Claimed both the gross and net income limits are exceeded. Wisconsin BBCE sets the gross limit at 200% FPL (about $2,608), which this household passes, and categorical eligibility waives the net income test. The household therefore gets the $24/month minimum allotment." -us,scenario_108,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly identified Wisconsin BBCE eligibility and the minimum-benefit outcome, but used the outdated $23 minimum. The FY2026 one-person minimum is $24/month (8% of the $298 maximum), so the annual amount is $288, not $276." -us,scenario_108,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Returned zero without applying the minimum-allotment rule. The household is categorically eligible through Wisconsin BBCE, so although the formula benefit is negative, the one-person household receives $24/month ($288/yr)." -us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Claimed $2,581.83/month gross income exceeds SNAP limits, but it passes Wisconsin's 200% FPL BBCE gross test. It also reduced the benefit to $0 instead of applying the $24/month minimum allotment for categorically eligible one-person households." -us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Applied a net income threshold and zeroed the benefit. Wisconsin BBCE categorical eligibility waives the net income test, and the one-person household receives the $24/month minimum allotment." -us,scenario_108,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum, but stopped at zero. A categorically eligible one-person household receives the FY2026 minimum allotment of $24/month instead of zero." -us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,Computed max allotment minus 30% of net income as negative and returned $0. It missed the minimum-benefit rule: categorically eligible one-person households under Wisconsin BBCE receive $24/month ($288/yr). -us,scenario_108,snap,glm-5.3,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test, and applied a shelter cap that does not apply to elderly/disabled households. Wisconsin BBCE waives the net income test, so the household is eligible and gets the $24/month minimum allotment." -us,scenario_108,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Submitted an unsupported $1,172 (about $98/month) regular benefit. Net income of about $2,036 makes the 30% contribution ($611) exceed the $298 maximum, so the formula benefit is zero. The household gets only the $24/month categorical-eligibility minimum ($288/yr)." -us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,Defaulted to zero without applying Wisconsin's BBCE pathway or the minimum-allotment rule. A categorically eligible one-person household with a zero formula benefit receives $24/month ($288/yr). -us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Concluded that income is too high to produce a positive allotment and returned zero. Under Wisconsin BBCE the household is categorically eligible, so the one-person minimum allotment of $24/month applies." -us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Estimated an unsupported $103/month regular benefit. Rent of $600 adds little or no shelter deduction against adjusted income of about $2,214, and net income of about $2,036 makes the 30% contribution ($611) exceed the $298 maximum. The household gets only the $24/month minimum ($288/yr)." -us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Applied the elderly/disabled 100% FPL net income limit and declared the household ineligible. Wisconsin BBCE waives the net income test, and the categorically eligible one-person household receives the $24/month minimum allotment." -us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Failed the household on the net income limit, missing that Wisconsin's TANF non-cash BBCE confers categorical eligibility that waives it. The one-person household receives the $24/month minimum allotment ($288/yr)." -us,scenario_108,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Applied the SNAP net income limit and returned zero. Under Wisconsin BBCE the household is categorically eligible, which waives the net income test and entitles the one-person household to the $24/month minimum." -us,scenario_108,snap,gpt-6-sol,llm_error,categorical_eligibility,False,Concluded that net income is too high for any allotment and returned zero. Categorically eligible one-person households get the minimum allotment of $24/month in FY2026 even when the formula benefit is negative. -us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"Asserted that income exceeds the SNAP limits after deductions. Wisconsin BBCE (200% FPL gross) makes the household categorically eligible, and the one-person minimum allotment of $24/month applies." -us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"Compared annual net income to 100% FPL and declared the household ineligible. Wisconsin BBCE waives the net income test for this household, which then receives the $24/month minimum allotment ($288/yr)." -us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,"Treated the 100% FPL net income test as binding. Wisconsin BBCE (gross income under 200% FPL) confers categorical eligibility that removes it, leaving the household with the $24/month one-person minimum allotment." -us,scenario_108,snap,grok-4.7,llm_error,categorical_eligibility,False,"Failed the household on the net income limit. It missed the Wisconsin BBCE pathway, which waives that limit, and the $24/month minimum allotment for categorically eligible one-person households." -us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Computed net income of about $2,218 and failed the household against the ~$1,305 net limit. Wisconsin BBCE categorical eligibility waives the net income test, and the one-person household receives the $24/month minimum allotment." -us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test to ~$2,225 net income and returned zero. The household is categorically eligible under Wisconsin BBCE, so the net test is waived and the $24/month one-person minimum applies." -us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Declared the household ineligible under the 100% FPL net income limit. Wisconsin's TANF non-cash BBCE waives that limit, and the one-person household receives the $24/month minimum allotment." -us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income far exceeds the one-person maximum, but returned $0. Categorically eligible one-person households receive the FY2026 minimum allotment of $24/month ($288/yr)." -us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross test and a 100% FPL net test, and omitted the earned income deduction. Wisconsin BBCE uses a 200% FPL gross limit that the household passes and waives the net test, so the household gets the $24/month minimum allotment." -us,scenario_108,snap,ox-alpha,llm_error,thresholds_rates,False,"Applied a 165% FPL gross income limit. Wisconsin BBCE sets the limit at 200% FPL (about $2,608/month), which $2,581.83 passes. The household is categorically eligible and receives the $24/month minimum allotment." -us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Noted that 'SNAP would be minimum' when the 30% contribution exceeded the maximum, then returned $0 anyway. Categorically eligible one-person households receive the $24/month minimum allotment ($288/yr)." -us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Asserted that countable income exceeds the SNAP limit. Wisconsin BBCE (200% FPL gross) makes the household categorically eligible regardless of the net income test, and it receives the $24/month one-person minimum allotment." -us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model stopped at a gross tax of $134.33 (3.5% of $3,838) and never subtracted Wisconsin's nonrefundable renter school property tax credit. On the $7,200 rent that credit is 12% of $1,440-$1,800, or $172.80-$216, which wipes out the tax. It also used a made-up $270 age deduction instead of Wisconsin's $700 personal exemption plus the $250 exemption for age 65 and over." -us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model said outright that there were 'no other nonrefundable credits', which skips the renter school property tax credit. That credit is 12% of 20-25% of the $7,200 rent, or $172.80-$216, and it takes the $161.71 gross tax to zero. It also used an old 3.54% bottom rate instead of the current 3.50% and left out the $700 and $250 exemptions." -us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied the bottom rate to about $4,548 of taxable income and reported the $159 gross tax as the final answer. It never subtracted the nonrefundable renter school property tax credit ($172.80-$216 on $7,200 rent), which brings Wisconsin tax before refundable credits to $0." -us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model did apply the renter school property tax credit but made it too small, leaving $70 of tax. The credit is 12% of rent treated as property tax (20-25% of the $7,200 rent, or $1,440-$1,800), which is $172.80-$216. That is more than the roughly $110-$150 gross tax, so the correct liability is zero." -us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model worked from an outdated $12,760 standard deduction with a phaseout and a $1,400 exemption, reached $146.45 of gross tax, and stopped. It never applied the nonrefundable renter school property tax credit ($172.80-$216 on $7,200 rent), which reduces the tax to $0." -us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model's reasoning covers only the standard deduction and personal exemption, and it reported $144.13 of gross tax at the bottom bracket as the answer. It left out the nonrefundable renter school property tax credit (12% of 20-25% of the $7,200 rent, or $172.80-$216), which is larger than that tax and brings it to $0." -us,scenario_108,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model taxed $3,538 at 3.5% to get $124 and stopped. It never subtracted Wisconsin's nonrefundable renter school property tax credit, which is $172.80-$216 on $7,200 of rent and is larger than $124, so the tax before refundable credits is $0." -us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said outright that there were 'no nonrefundable state credits', so it skipped the renter school property tax credit ($172.80-$216 on $7,200 rent) that takes its $150 tax to zero. It also used an old 3.54% bottom rate instead of 3.50% and left out the $700 and $250 exemptions." -us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to grade. The correct derivation takes a small gross Wisconsin tax and applies the nonrefundable renter school property tax credit, which brings it to $0." -us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 in household income instead of $20,590 (wages plus Social Security). It also took 25% of rent capped at $1,460 instead of 20% of rent ($1,440). Its $636 does not follow from any formula it stated; the correct calculation, 0.8 × ($1,440 − 8.785% × $12,530), gives $271.39." -us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It added the separate $10,392 survivor benefits, putting household income at $30,982 and over the $24,680 limit. Homestead household income is wages plus Social Security, $20,590, which qualifies for a $271.39 credit." -us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It correctly found $20,590 in income but wrongly said the Homestead Credit is not a refundable income tax credit. The credit is claimed on the Wisconsin return and is refundable: 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It reasoned that refundable credits require Wisconsin tax liability to offset. The Homestead Credit is paid in full even with no liability, and this 85-year-old disabled renter with $20,590 in household income gets $271.39." -us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It wrongly left Social Security out of household income and wrongly added the $10,392 survivor benefits, getting $18,048 instead of $20,590. It then guessed $300 rather than applying 0.8 × ($1,440 − 8.785% × (income − $8,060)), even though it used the correct $1,440 rent figure." -us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It counted the separate $10,392 survivor benefits in household income ($30,982 total) and concluded the household was over the $24,680 limit. Wages plus Social Security total $20,590, which gives a $271.39 credit." -us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Its income of about $18,000 means it dropped the $12,934 in Social Security and counted the $10,392 survivor benefits. The correct $20,590 gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39, not $371." -us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, making household income $30,982 instead of $20,590, and wrongly ruled out the credit under the $24,680 limit." -us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It added the $10,392 in other survivor benefits to wages and Social Security, getting $30,982. Homestead household income here is $20,590, which is under the limit and gives a $271.39 credit." -us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It dismissed the Homestead Credit with a vague appeal to income level and age and never calculated it. An 85-year-old disabled renter with $20,590 in household income and $1,440 in rent-based property taxes qualifies for $271.39." -us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It used the correct $20,590 income but took 25% of rent capped at $1,460 instead of 20% of rent ($1,440 when heat is not included). It also used an 8.65% rate and an $8,000 threshold instead of 8.785% and $8,060, which gave $297 instead of $271.39." -us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit, which only happens if the $10,392 survivor benefits are counted. Wages plus Social Security total $20,590, which is under the $24,680 limit and gives $271.39." -us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Its $30,982 income figure includes the separate $10,392 survivor benefits. Homestead household income is $20,590, which qualifies for a $271.39 credit." -us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit because it counted the $10,392 survivor benefits on top of wages and Social Security. The correct $20,590 gives a $271.39 credit." -us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It never considered the Homestead Credit, which this 85-year-old disabled renter with $20,590 in household income and $7,200 in rent qualifies for. It is worth 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It put household income at $30,982 by including the $10,392 survivor benefits. Wages plus Social Security total $20,590, which is under the $24,680 limit and gives $271.39." -us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It said the household was ineligible without calculating anything and skipped the Homestead Credit. With $20,590 in household income and $1,440 in rent-based property taxes, the credit is $271.39." -us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Its $30,982 total includes the separate $10,392 survivor benefits. Homestead household income is $20,590 (wages plus Social Security), which qualifies for $271.39." -us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, got $30,982 and wrongly ruled out the credit under the $24,680 limit. The correct $20,590 gives a $271.39 credit." -us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It said the head was ineligible for the Homestead Credit without any income test. The head is 85 and disabled, has $20,590 in household income and pays $7,200 in rent, which gives $271.39." -us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in household income ($30,982) instead of using wages plus Social Security ($20,590). The correct figure is under the limit and gives $271.39." -us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit, which requires counting the $10,392 survivor benefits. Homestead household income is $20,590 and the credit is $271.39." -us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It said income was over the $24,680 limit by including the separate survivor benefits. Wages plus Social Security total $20,590, which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." -us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It used $30,982 as household income by adding the $10,392 survivor benefits. The correct base is $20,590, which qualifies for a $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as pension income in homestead household income, getting $30,982. Homestead household income is $20,590 (wages plus Social Security), which gives a $271.39 credit." -us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It never looked at the Homestead Credit, even though this elderly, disabled renter qualifies. With $20,590 in household income and $1,440 in rent-based property taxes, the credit is $271.39." -us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,It looked only for EITC-type refundable credits and missed the Homestead Credit. This 85-year-old disabled renter qualifies for $271.39. -us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It counted both the Social Security and the separate $10,392 survivor benefits, pushing household income over the limit. Wages plus Social Security total $20,590, which gives $271.39." -us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It wrongly concluded that no refundable credit applies to an elderly filer without children and missed the Homestead Credit. That credit pays elderly or disabled renters with income under $24,680, here $271.39 on $20,590 of income." -us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It said income was over the Homestead Credit limit, which only holds if the $10,392 survivor benefits are counted. Homestead household income is $20,590, which gives a $271.39 credit." -us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It saw no sign of eligibility and missed that the Homestead Credit applies to an 85-year-old disabled renter paying $7,200 in rent. With $20,590 in household income the credit is $271.39." -us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits, got $30,982 and compared that with a wrong $24,480 limit. Correct household income is $20,590, which qualifies for $271.39." -us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It said household income was over the Homestead Credit limit by including the separate survivor benefits. Wages plus Social Security total $20,590 and give a $271.39 credit." -us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"Its total income figure counted the $10,392 survivor benefits on top of wages and Social Security. Homestead household income is $20,590, under the limit, which gives $271.39." -us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It counted all three income streams against the Homestead Credit limit. The base is $20,590 (wages plus Social Security), which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." -us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It gave no analysis and missed the Homestead Credit, for which this elderly, disabled renter with $20,590 in household income qualifies at $271.39." -us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 in household income. The correct $20,590 is under the limit and gives $271.39." -us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It included the separate $10,392 survivor benefits, putting household income at $30,982 instead of $20,590, and so wrongly ruled out a $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly counted the other survivor benefits in homestead household income. Wages plus Social Security total $20,590, which gives 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Its $30,982 figure includes the $10,392 survivor benefits, and it compared that with a made-up limit of about $26,500. Correct household income is $20,590, which gives $271.39." -us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It checked only the Wisconsin earned income credit and missed the Homestead Credit. This 85-year-old disabled renter with $20,590 in household income gets $271.39." -us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_refundable_credits, so there was nothing to score against the $271.39 Homestead Credit." -us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a broad household income of $30,982 that includes the separate $10,392 survivor benefits. Homestead household income is $20,590, which qualifies for $271.39." -us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,It wrongly assumed refundable credits require Wisconsin tax liability. The Homestead Credit is paid regardless of liability and comes to $271.39 for this renter. -us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 other survivor benefits in household income, getting $30,982. The correct $20,590 is under the limit and gives a $271.39 credit." -us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It summed wages, Social Security and the separate $10,392 survivor benefits to get $30,982. Homestead household income is $20,590, which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." -us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,It tied refundable credits to having qualifying children and missed the Homestead Credit. That credit is available to childless elderly or disabled renters and is $271.39 here. +us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test and called the household ineligible. It never recognized that Wisconsin BBCE (200% FPL gross limit, which $2,582 passes) makes the household categorically eligible, so the net income test does not apply and the one-person household gets the $24/month minimum allotment ($288/year)." +us,scenario_108,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Concluded ineligibility because net income of about $2,214 exceeds 100% FPL. It missed Wisconsin's BBCE pathway, which waives the net income test for a household under 200% FPL gross, and the guaranteed $24/month minimum allotment for categorically eligible one-person households." +us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied a 130% FPL limit, which does not apply to this elderly/disabled household under Wisconsin's 200% FPL BBCE threshold. That test produced $0, but categorical eligibility guarantees the $24/month minimum allotment ($288/year)." +us,scenario_108,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Gave $0 without any benefit computation. It ignored that the household is categorically eligible through Wisconsin BBCE, so even though 30% of net income exceeds the maximum allotment, the one-person household receives the $24/month minimum ($288/year)." +us,scenario_108,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly worked out that the formula benefit is negative and that a minimum benefit applies, but it used the FY2025 $23 minimum instead of the FY2026 $24 (8% of the $298 maximum allotment). It then abandoned that result and submitted an unsupported $2,227, not the $288 the minimum-benefit logic produces." +us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Its reasoning failed the household on the 100% FPL net income test without recognizing Wisconsin BBCE categorical eligibility, which guarantees the $24/month minimum. It then submitted $2,376, a figure its own reasoning does not support, instead of $288." +us,scenario_108,snap,claude-opus-5,llm_error,thresholds_rates,False,"Used the FY2025 $23/month minimum allotment instead of the FY2026 $24 (8% of the $298 one-person maximum), giving $276 instead of $288. It also overstated the medical deduction ($465/month instead of about $32), when the minimum actually applies because of BBCE categorical eligibility, not because deductions shrink net income." +us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Stated outright that Wisconsin has no broad-based categorical eligibility, which is false: WI FoodShare uses BBCE with a 200% FPL gross limit. That error led it to apply the net income test and answer $0 instead of the $24/month minimum for a categorically eligible one-person household." +us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Invented a net income of about $1,700 and a $164/month benefit, but $600 rent is below half of adjusted income, so there is no excess shelter deduction. Net income stays about $2,036, the contribution of about $611 exceeds the $298 maximum, and only the $24 categorical-eligibility minimum applies ($288/year)." +us,scenario_108,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Applied only the 100% FPL net income test and declared the household ineligible. It missed that Wisconsin BBCE makes the household categorically eligible (gross under 200% FPL), which removes the net test and guarantees the $24/month minimum allotment." +us,scenario_108,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Failed the household on the 100% FPL net income limit. It did not apply Wisconsin BBCE categorical eligibility, under which the net test is waived and a one-person household receives the $24/month minimum allotment ($288/year)." +us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,Correctly found that the formula benefit is negative but floored it at $0. It omitted the minimum-benefit rule: categorically eligible one- and two-person households receive 8% of the one-person maximum ($24/month in FY2026). +us,scenario_108,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,Correctly found that 30% of net income exceeds the one-person maximum allotment but set the benefit to zero. It skipped the minimum allotment ($24/month) guaranteed to categorically eligible one-person households under Wisconsin BBCE. +us,scenario_108,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Treated the 100% FPL net income test as decisive. Wisconsin BBCE confers categorical eligibility (gross income under 200% FPL), which bypasses that test and yields the $24/month minimum allotment." +us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Treated $2,581.83 gross income as over the limits, but it is under Wisconsin's 200% FPL BBCE limit ($2,608.33). It also floored the negative formula benefit at $0 instead of applying the $24/month minimum for categorically eligible one-person households." +us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Asserted the household is over the income limit, ignoring that gross income is under Wisconsin's 200% FPL BBCE threshold. Categorical eligibility guarantees the $24/month minimum allotment ($288/year)." +us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Claimed gross and net income exceed SNAP limits. Under Wisconsin BBCE the gross limit is 200% FPL ($2,608.33, which $2,581.83 passes) and the net test is waived, so the one-person household receives the $24/month minimum." +us,scenario_108,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Correctly identified Wisconsin's 200% FPL BBCE pathway and the minimum benefit, but it used the FY2025 $23 minimum instead of the FY2026 $24 (8% of the $298 one-person maximum allotment). That gives $276 instead of $288." +us,scenario_108,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Returned $0 without applying the minimum-benefit rule. As a BBCE categorically eligible one-person household in Wisconsin, it receives $24/month even though the formula benefit is negative." +us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Claimed $2,581.83 gross income exceeds the SNAP limits, but it falls under Wisconsin's 200% FPL BBCE limit. It also floored the negative benefit at $0 instead of paying the $24/month categorical-eligibility minimum." +us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Treated net income above the threshold as disqualifying. Wisconsin BBCE categorical eligibility waives the net income test, and the one-person household receives the $24/month minimum allotment." +us,scenario_108,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the maximum allotment, then stopped at $0. Categorically eligible one-person households receive the minimum allotment of 8% of the maximum ($24/month in FY2026)." +us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,"Computed a negative formula benefit (about $300 minus $659) and floored it at $0. It omitted the minimum allotment for categorically eligible one-person households, which is $24/month under Wisconsin BBCE." +us,scenario_108,snap,glm-5.3,llm_error,categorical_eligibility,False,"Applied a 100% FPL net income limit and a capped shelter deduction, which does not apply to an elderly/disabled household. It missed that Wisconsin BBCE categorical eligibility waives the net test and guarantees the $24/month minimum." +us,scenario_108,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted eligibility with a $1,172 benefit without computing it. Net income of about $2,036 makes the 30% contribution (about $611) exceed the $298 maximum, so the categorically eligible household receives only the $24/month minimum ($288/year)." +us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to $0 without running the eligibility pathway. The household is categorically eligible through Wisconsin BBCE (gross under 200% FPL), so the one-person household receives the $24/month minimum allotment." +us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Correctly found that no positive formula allotment results after deductions, but set SNAP to zero. Categorically eligible one- and two-person households get the minimum allotment, which is $24/month in FY2026." +us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Produced $103/month by overstating deductions. Rent of $600 is below half of adjusted income, so there is no shelter deduction, net income stays about $2,036, and the formula benefit is negative. Only the $24/month BBCE minimum applies." +us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,Applied the elderly/disabled net income limit and declared the household ineligible. Wisconsin BBCE categorical eligibility (gross under 200% FPL) waives that limit and guarantees the $24/month minimum allotment. +us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Failed the household on the elderly/disabled net income limit. It missed that Wisconsin BBCE makes the household categorically eligible, which bypasses the net test and yields the $24/month minimum." +us,scenario_108,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Treated net income above the SNAP net limit as disqualifying. Under Wisconsin BBCE the household is categorically eligible, so the net test does not apply and the one-person minimum of $24/month is paid." +us,scenario_108,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Correctly concluded that net income is too high for a positive formula allotment, but floored the benefit at $0. It omitted the $24/month minimum allotment guaranteed to categorically eligible one-person households." +us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"Asserted the household is over the income limits. Wisconsin BBCE (200% FPL gross) makes it categorically eligible, and a one-person categorically eligible household receives the $24/month minimum allotment." +us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"Compared net income to 100% FPL and returned $0. Wisconsin BBCE categorical eligibility waives the net income test, and the minimum allotment of $24/month applies ($288/year)." +us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,Correctly waived the gross test for an elderly/disabled household but then applied the 100% FPL net test. It missed that Wisconsin BBCE categorical eligibility removes the net test and guarantees the $24/month minimum. +us,scenario_108,snap,grok-4.7,llm_error,categorical_eligibility,False,"Treated the one-person net income limit as binding. Under Wisconsin BBCE the household is categorically eligible (gross under 200% FPL), so it bypasses the net limit and receives the $24/month minimum allotment." +us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Calculated net income of about $2,218 against a $1,305 net limit and declared ineligibility. It missed Wisconsin's BBCE categorical eligibility, which waives the net test and pays the $24/month minimum to one-person households." +us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,"Failed the household on the 100% FPL net income test. Wisconsin BBCE confers categorical eligibility at gross income under 200% FPL, so the net test is waived and the $24/month minimum allotment applies." +us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit as disqualifying. Wisconsin BBCE categorical eligibility removes that test, and the one-person household receives the $24/month minimum allotment ($288/year)." +us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"Correctly found that the roughly $664 contribution exceeds the maximum allotment, but floored the benefit at $0. Categorically eligible one-person households receive a minimum allotment of 8% of the maximum ($24/month in FY2026)." +us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross limit, which does not apply, and a net limit of about $1,180. Wisconsin BBCE uses a 200% FPL gross limit ($2,608.33) and waives the net test, so the household is categorically eligible for the $24/month minimum." +us,scenario_108,snap,muse-wonder,llm_error,categorical_eligibility,False,"Treated net income above the one-person limit as disqualifying. Wisconsin BBCE makes the household categorically eligible, so the net limit does not apply and the $24/month minimum allotment is paid." +us,scenario_108,snap,ox-alpha,llm_error,categorical_eligibility,False,"Used a 165% FPL gross limit (about $2,150), but Wisconsin's BBCE gross limit is 200% FPL ($2,608.33), which the $2,581.83 income passes. The household is categorically eligible for the $24/month minimum allotment." +us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Noted at one point that 'SNAP would be minimum' when the contribution exceeds the maximum, but then concluded $0. It never applied the categorical-eligibility minimum of 8% of the FY2026 one-person maximum ($24/month), and it used an outdated $292 maximum." +us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Asserted countable income exceeds the SNAP limit. Wisconsin BBCE categorical eligibility (gross under 200% FPL) waives the net income limit, and the one-person household receives the $24/month minimum allotment." +us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin income of $18,048 by counting the $10,392 survivor benefits as taxable. The engine's taxable base is only the $7,656 in wages, which the $13,940 standard deduction fully absorbs, so tax is $0. The model also left out the nonrefundable renter school property tax credit (about $173 on $7,200 of rent), which alone more than cancels its $134.33." +us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model taxed an $18,048 base that wrongly includes the $10,392 survivor benefits. It also skipped the $700 Wisconsin personal exemption and never applied the nonrefundable renter school property tax credit (about $173 on $7,200 of rent). With wages of $7,656 as the only taxable income, the standard deduction leaves $0 tax, not $161.71." +us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model counted the $10,392 survivor benefits in a Wisconsin AGI of $18,048 and taxed the leftover $4,548 at 3.5%. The engine's Wisconsin base after the Social Security exemption is only the $7,656 in wages, which falls below the standard deduction. The renter school property tax credit (about $173 on $7,200 of rent) would also have wiped out the $159." +us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model counted the $10,392 survivor benefits in its $18,048 Wisconsin AGI, which inflated pre-credit tax enough that $70 was left after the renter school property tax credit. Wisconsin taxable income is only the $7,656 in wages. The standard deduction and exemptions reduce that to $0, so there is no tax before credits." +us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 survivor benefits as taxable, giving $18,048 of income, and used a $1,400 personal exemption instead of Wisconsin's $700 plus $250 for age 65+. It never applied the renter school property tax credit (about $173 on $7,200 of rent). With only $7,656 in taxable wages, the standard deduction leaves $0 tax, not $146.45." +us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly taxed the survivor-benefit income alongside wages. The engine's Wisconsin base is only the $7,656 in wages after Social Security is exempted, and the standard deduction fully covers it. The model also ignored the nonrefundable renter school property tax credit on $7,200 of rent, which more than offsets its $144.13." +us,scenario_108,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's $3,538 of Wisconsin taxable income only works if the $10,392 survivor benefits are counted as taxable. With just $7,656 of taxable wages, the standard deduction and exemptions leave $0. The model also never subtracted the renter school property tax credit (about $173), which exceeds its $124 by itself." +us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included the $10,392 survivor benefits in its $18,048 Wisconsin AGI and stated outright that there were no nonrefundable state credits. Wisconsin's school property tax credit for renters applies here: 12% of 20–25% of the $7,200 rent comes to roughly $173–$216. Together with the correct wage-only base of $7,656, tax before refundable credits is $0, not $150." +us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or reasoning for state_income_tax_before_refundable_credits, so there was no answer to score against the $0 reference." +us,scenario_108,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model taxed wages plus the $10,392 survivor income, leaving $2,680 taxable at 3.5%. The engine's Wisconsin base is only the $7,656 in wages, which is below the standard deduction, so tax is $0. The model also skipped the renter school property tax credit, which by itself cancels the $94." +us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits and got $30,982 of household income. It then guessed $636 with no formula behind it, using 25% of rent capped at $1,460. The correct inputs are $20,590 of income (wages plus Social Security) and $1,440 of property taxes (20% of rent), giving 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It added the separate $10,392 survivor benefits to homestead household income ($30,982) and declared the household over the $24,680 limit. Household income is only wages plus Social Security, $20,590, which qualifies for a $271.39 homestead credit." +us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It computed the correct $20,590 of income but said the Wisconsin Homestead Credit is not a refundable income tax credit. It is refundable and is claimed on the state return, and at $20,590 of income and $1,440 of property taxes accrued it pays $271.39." +us,scenario_108,state_refundable_credits,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It assumed a refundable credit needs Wisconsin tax liability to offset and never considered the Homestead Credit. That credit is paid regardless of liability: $271.39 at $20,590 of income and $1,440 of property taxes accrued." +us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It used the correct $1,440 for property taxes accrued, but built household income backwards. It left out Social Security, which counts, and included the $10,392 survivor benefits, which do not, getting $18,048 instead of $20,590. It then guessed $300 instead of applying 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to wages and Social Security to get $30,982, above the $24,680 limit. Homestead household income is $20,590 (wages plus Social Security), which yields a $271.39 credit." +us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Its income figure of about $18,000 means it dropped Social Security and kept the survivor benefits, and it guessed $371 without the formula. Household income is $20,590 (wages plus Social Security), and the credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in homestead household income ($30,982) and wrongly found the household over the $24,680 limit. The correct $20,590 (wages plus Social Security) produces a $271.39 credit." +us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It added the separate $10,392 survivor benefits to homestead household income to reach $30,982 and denied the credit. Household income is only wages plus Social Security, $20,590, which is under the $24,680 ceiling and yields $271.39." +us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It named the Homestead Credit but dismissed it without running the computation, assuming only EITC-type credits could apply. An 85-year-old disabled renter with $20,590 of household income and $1,440 of property taxes accrued gets $271.39." +us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It used the correct $20,590 of income but took 25% of rent ($1,800, capped at $1,460) instead of PolicyEngine's 20% ($1,440). It also used an outdated $8,000 threshold and 8.65% rate instead of $8,060 and 8.785%. With the right figures, 0.8 × ($1,440 − 0.08785 × $12,530) = $271.39, not $297." +us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It said household income was above the Homestead Credit limit, which only happens if the $10,392 survivor benefits are added. Homestead income is wages plus Social Security, $20,590, which qualifies for $271.39." +us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits in homestead income ($30,982) and denied the credit. The correct $20,590 (wages plus Social Security) is under the limit and gives $271.39." +us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the Homestead Credit limit, which only follows from including the $10,392 survivor benefits. Homestead household income is $20,590 (wages plus Social Security), which yields a $271.39 credit." +us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It gave a bare 'no qualifying credits' and never evaluated the refundable Wisconsin Homestead Credit. At $20,590 of household income and $1,440 of property taxes accrued, that credit is $271.39." +us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It put homestead income at $30,982 by adding the $10,392 survivor benefits on top of wages and Social Security. The correct figure is $20,590, under the $24,680 ceiling, giving a $271.39 credit." +us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It denied all Wisconsin refundable credits without evaluating the Homestead Credit, for which this 85-year-old disabled renter qualifies. With $20,590 of household income and $1,440 of property taxes accrued, the credit is $271.39." +us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in household income ($30,982) and found the household ineligible. Homestead income is $20,590 (wages plus Social Security), which produces $271.39." +us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted the separate $10,392 survivor benefits, got $30,982, and compared that with the $24,680 limit. The correct homestead income is $20,590, which qualifies for a $271.39 credit." +us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It declared the head ineligible for the Homestead Credit without any income test. With $20,590 of household income (under $24,680) and $1,440 of property taxes accrued, the credit is $271.39." +us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to household income ($30,982) and denied the credit for exceeding $24,680. Homestead income is only wages plus Social Security, $20,590, which yields $271.39." +us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the Homestead limit, which requires counting the $10,392 survivor benefits. The correct $20,590 (wages plus Social Security) gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It placed household income above the $24,680 Homestead limit by including the $10,392 survivor benefits. Homestead income is $20,590 (wages plus Social Security), which produces a $271.39 credit." +us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It used $30,982 of household income, which includes the $10,392 survivor benefits, and denied the Homestead Credit. Correct homestead income is $20,590, which qualifies for $271.39." +us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as countable pension income, reached about $30,982, and denied the credit. Homestead household income is only wages plus Social Security, $20,590, which yields $271.39." +us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It said no Wisconsin refundable credit was triggered, overlooking the Homestead Credit for a renter aged 62 or older. At $20,590 of household income and $1,440 of property taxes accrued, it pays $271.39." +us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It looked only for EITC-type refundable credits and never evaluated the Wisconsin Homestead Credit. That credit is $271.39 at $20,590 of household income and $1,440 of property taxes accrued." +us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It included the listed survivor benefits in homestead household income and found the household over the limit. Household income is $20,590 (wages plus Social Security), under $24,680, which gives a $271.39 credit." +us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It concluded no refundable credit applies to an elderly filer without children, ignoring the Homestead Credit. That credit exists specifically for low-income elderly renters and pays $271.39 here." +us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It said income was above the homestead limit, which only follows from adding the $10,392 survivor benefits. Homestead household income is $20,590 (wages plus Social Security), which produces $271.39." +us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It found no refundable credit eligibility and skipped the Homestead Credit, which an 85-year-old disabled renter qualifies for. With $20,590 of income and $1,440 of property taxes accrued, the credit is $271.39." +us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits to reach $30,982 and also used a wrong $24,480 ceiling. Homestead income is $20,590 (wages plus Social Security), under the $24,680 limit, which yields $271.39." +us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the homestead limit, which comes from counting the $10,392 survivor benefits. Correct homestead income is $20,590, which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It put total household income over the homestead limit by including the $10,392 survivor benefits. Homestead income is wages plus Social Security, $20,590, which qualifies for $271.39." +us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It treated the household as over the homestead limit by counting the separate survivor benefits. Homestead household income is $20,590, under $24,680, which yields a $271.39 credit." +us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It gave a bare 'no credits qualify' and never evaluated the Wisconsin Homestead Credit. With $20,590 of household income and $1,440 of property taxes accrued, that credit is $271.39." +us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in household income (about $30,982) and denied the Homestead Credit. Correct homestead income is $20,590 (wages plus Social Security), which gives $271.39." +us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It added both the Social Security and the separate $10,392 survivor benefits to get about $30,982, above the limit. Homestead income is only wages plus Social Security, $20,590, which yields $271.39." +us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly counted the other survivor benefits along with wages and Social Security, pushing income over the homestead limit. Household income is $20,590, which qualifies for 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as part of AGI plus Social Security ($30,982) and also used a made-up $26,500 limit. Homestead income is $20,590, under the $24,680 ceiling, which yields $271.39." +us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It checked only the Wisconsin EITC and said no other refundable credits are claimed, missing the Homestead Credit. The engine assumes take-up, and the credit is $271.39 at $20,590 of income and $1,440 of property taxes accrued." +us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no usable value or explanation for state_refundable_credits. The correct derivation is a Homestead Credit of 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It correctly identified the Homestead Credit as refundable but used a broad household income of about $30,982 that includes the $10,392 survivor benefits. Homestead income is $20,590 (wages plus Social Security), which produces $271.39." +us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,"It assumed refundable credits require Wisconsin tax liability. The Homestead Credit is paid without any liability: $271.39 at $20,590 of household income and $1,440 of property taxes accrued." +us,scenario_108,state_refundable_credits,muse-wonder,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the homestead limit, which requires counting the $10,392 survivor benefits. Correct homestead income is $20,590 (wages plus Social Security), which gives $271.39." +us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It explicitly added the other survivor benefits to wages and Social Security ($30,982) and denied the credit. Homestead household income is $20,590, under $24,680, which yields $271.39." +us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It summed wages, Social Security survivor benefits and the separate $10,392 survivor benefits to get $30,982 and denied the credit. Homestead income is only $7,656 + $12,934 = $20,590, which gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It tied Wisconsin refundable credits to qualifying children and ignored the Homestead Credit, which needs no children. An 85-year-old renter with $20,590 of household income gets $271.39." us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income limits and treated satisfying them as sufficient. It omitted the prior Medicaid screen: the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, which makes the child ineligible for CHIP." us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model calculated MAGI and compared it only with CHIP's upper-income limit. It failed to test the lower Medicaid eligibility pathway first; at this income, the age-9 child is Medicaid-eligible in Florida and therefore excluded from CHIP." us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model recognized that CHIP applies between Medicaid and CHIP income limits but incorrectly placed this household in that interval. The age-9 child remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child never enters the CHIP coverage band." @@ -8724,47 +8834,48 @@ us,scenario_109,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_elig us,scenario_109,child3_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified the infant category and computed income at roughly 0.79 FPL against a ~200% FPL Florida infant limit, then reversed its own conclusion by imposing a Florida asset test and requiring Medicaid expansion or confirmed enrollment. MAGI-based children's categories carry no asset or resource test, non-expansion status restricts only the adult expansion group, and CHIP is residual coverage for children who fail Medicaid rather than a substitute for it — so the infant category grants eligibility, and the question asked for eligibility rather than enrollment." us,scenario_109,child3_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed WIC status as disqualifying instead of computing eligibility from the supplied facts. The age-zero child satisfies WIC's child category, and the five-person household's income is below the 185%-of-poverty income limit, so the correct eligibility output is Yes." us,scenario_109,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an overstated projected standard deduction, incorrectly concluded taxable income and pre-credit tax were zero, and then subtracted an invented $284 of “unused” CTC. The correct computation produces income tax that uses $2,792.91 of the $6,600 CTC as a nonrefundable offset, while the remaining $3,807.09 is refundable; unused nonrefundable credits never make this output negative." -us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived EITC $8,231 plus ACTC $3,807 for a total of $12,038. It then rounded the submitted value to $12,000, which threw away $38.09 of a computed exact amount." -us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model claimed that $28k of income was above the EITC phase-out, so it gave no EITC. In fact, $27,880.57 of earned income is on the MFJ three-child plateau and earns the full $8,231. It also computed an ACTC of $3,877.50 and then submitted an arbitrary $2,500 instead." -us,scenario_109,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model treated $30,000 of self-employment income as producing no EITC and roughly zero refundable CTC. SE profit minus half of SE tax ($27,880.57) is earned income for both credits. It produces the full $8,231 EITC and an ACTC of $3,807.09." -us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's own arithmetic reached the correct total: EITC $8,231 + ACTC $3,807 = $12,038. It then submitted an unsupported $10,379 that matches none of its intermediate figures." -us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the 2025 EITC maximum ($8,046) instead of the 2026 value ($8,231). It computed the ACTC on $27,705 of net SE earnings rather than on $27,880.57 (profit minus half of SE tax). It then replaced its $3,781 ACTC with an unexplained $4,416 in the final sum." -us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model placed the household in the EITC phase-out and cut the credit to about $5,035. Earned income of $27,880.57 is below the MFJ three-child phase-out start, so the full $8,231 applies. It also computed the ACTC on gross $30,000 ($4,125) instead of on earned income net of the half-SE-tax deduction ($3,807.09)." -us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied an EITC phase-out starting near $26.5k, which is the wrong threshold. The MFJ three-child phase-out begins above $30k, so the full $8,231 applies. It computed the ACTC on gross $30,000 instead of $27,880.57. Finally, it dropped the thousands digit of its own $11,594 total and submitted $7,594." -us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model cut the EITC to about $3,015 as if it were deep in the phase-out. At $27,880.57 of earned income, a three-child MFJ household is on the plateau at $8,231. It also set the ACTC to the $5,100 per-child cap even though its own 15%-of-earnings-over-$2,500 limit gave $3,807." -us,scenario_109,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model computed the ACTC as 15% of gross $30,000 minus $2,500 ($4,125). Earned income for the ACTC is SE profit reduced by the deductible half of SE tax ($27,880.57), which gives $3,807.09. It also used $8,215 for the 2026 EITC maximum instead of $8,231." -us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model set the refundable CTC at $3,000 ($1,000 per child), as if the TCJA had expired. The 2026 CTC allows up to $1,700 refundable per child, limited to 15% of earnings over $2,500, which gives $3,807.09. It also understated the 2026 EITC maximum as $8,026 instead of $8,231." -us,scenario_109,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The ACTC of $3,807 is correct. The model overstated the 2026 three-child EITC maximum as $8,261 instead of $8,231, so its total is $30 too high." -us,scenario_109,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model computed the ACTC on gross $30,000 ($4,125) instead of on earned income net of the half-SE-tax deduction ($27,880.57 → $3,807.09). It also overstated the 2026 EITC maximum as $8,290 instead of $8,231." -us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $1,000-per-child ACTC ($3,000) from expired pre-TCJA law. The 2026 refundable cap is $1,700 per child, and the binding 15% earned-income formula gives $3,807.09. It also overstated the EITC maximum as $8,290 instead of $8,231." -us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model cut the EITC to $3,588 as if it were deep in the phase-out. The household's $27,880.57 of earned income is on the three-child MFJ plateau, so the EITC is $8,231. Its $1,674 refundable CTC also understates 15% × ($27,880.57 − $2,500) = $3,807.09." -us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a flat $3,000 ACTC ($1,000 per child) instead of the 2026 formula. That formula, 15% of earned income over $2,500 with a $1,700-per-child cap, gives $3,807.09. It also overstated the EITC maximum as $8,250 instead of $8,231." -us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated the maximum ACTC as $1,000 per child ($3,000). Under 2026 law the cap is $1,700 per child, and the earned-income formula gives $3,807.09. It also set the EITC maximum at $8,240 instead of $8,231." -us,scenario_109,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no derivation. Its $7,502 is $4,536 below the correct total of the full $8,231 plateau EITC plus the $3,807.09 ACTC (15% of the $27,880.57 earned income above $2,500), so it badly under-counted at least one of the two credits." -us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $3,000 ACTC ($1,000 per child) from expired-TCJA law instead of the 2026 formula, which yields $3,807.09. It also used $8,219 for the EITC maximum instead of $8,231." -us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $11,046 total matches the 2025 EITC maximum ($8,046) plus a $1,000-per-child ACTC ($3,000). The correct figures are the 2026 maximum of $8,231 and an ACTC of 15% × ($27,880.57 − $2,500) = $3,807.09." -us,scenario_109,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied a $1,000-per-child ACTC ($3,000) instead of the 2026 formula, which yields $3,807.09 under the $1,700-per-child cap. It also understated the EITC maximum as $8,207 instead of $8,231." -us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model understated the 2026 EITC maximum as $8,025.75 instead of $8,231. It computed the ACTC on $27,705 (the 0.9235 SE-tax base) instead of on SE profit minus half of SE tax ($27,880.57), which gave $3,780.75 rather than $3,807.09." -us,scenario_109,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model reduced earned income twice. It applied the 0.9235 factor and also subtracted half of SE tax, getting $25,586 instead of $30,000 − $2,119.43 = $27,880.57. That cut the ACTC to $3,463 instead of $3,807.09. It also used $8,042 for the EITC maximum instead of $8,231." -us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no computation and submitted $13,400, which overstates the total. The correct total is the $8,231 maximum EITC plus an ACTC limited to 15% of earned income over $2,500, or $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model refused to count self-employment income as earned income for the EITC and ACTC. SE profit minus half of SE tax ($27,880.57) is earned income under IRC §32(c)(2)(A)(ii), and it qualifies the household for an $8,231 EITC and a $3,807.09 ACTC." -us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The EITC of $8,231 is correct. The model used a $1,000-per-child ACTC ($3,000) as if the TCJA had expired. The 2026 CTC has a $1,700-per-child refundable cap, so the earned-income formula governs and yields $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The EITC of $8,231 is correct. The ACTC was computed on gross $30,000 ($4,125) instead of on earned income reduced by the deductible half of SE tax ($27,880.57), which gives $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The EITC of $8,231 is correct. The model computed the ACTC on $27,705 (the 92.35% SE-tax base) instead of on SE profit minus the deductible half of SE tax ($27,880.57). That gave $3,781 instead of $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The EITC of $8,231 is correct. The ACTC of $4,125 uses gross $30,000 as earned income. It should use $30,000 minus the $2,119.43 half-SE-tax deduction, which yields $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The EITC of $8,231 is correct. The model computed the ACTC as 15% of gross SE income over $2,500 ($4,125) without first subtracting the deductible half of SE tax. The correct figure is 15% × ($27,880.57 − $2,500) = $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The ACTC of $3,807.09 is exactly right. The model overstated the 2026 three-child EITC maximum as $8,427 instead of $8,231, so its total is $196 too high." -us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model computed no EITC or ACTC, although $27,880.57 of SE earned income and three qualifying children produce a full $8,231 EITC and a $3,807.09 ACTC." -us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied expired-TCJA rules: a $1,000-per-child CTC and a $3,000 earnings threshold. For 2026, the CTC has a $1,700 refundable cap per child and a $2,500 threshold, which yields $3,807.09. It also used $8,226 for the EITC maximum instead of $8,231." -us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model treated the 2026 CTC as $1,000 per child with a $3,000 threshold, capping the ACTC at $3,000. Current law sets a $1,700 refundable cap and a $2,500 threshold, so the ACTC is $3,807.09. It also overstated the EITC maximum as $8,247 instead of $8,231." -us,scenario_109,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The EITC of $8,231 is correct. The model assumed the TCJA had expired, giving a $1,000-per-child CTC and a $3,000 threshold. The 2026 CTC keeps the $2,500 threshold and a $1,700-per-child refundable cap, so the ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09." -us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model capped the ACTC at $3,000 ($1,000 per child, with a $3,000 threshold) under expired-TCJA rules instead of the 2026 $1,700 cap and $2,500 threshold, which gives $3,807.09. It also overstated the EITC maximum as $8,245 instead of $8,231." -us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The ACTC of about $3,807 is correct. The model overstated the 2026 three-child EITC maximum as $8,271 instead of $8,231, so its total is $40 too high." -us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The model gave no reasoning. Its $6,695 is $5,343 below the correct total of the full $8,231 plateau EITC plus a $3,807.09 ACTC, so it missed most of one credit or wrongly phased down the EITC." -us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model assumed an EITC phase-out start near $29,600 and cut the credit to $3,588. Earned income of $27,880.57 is on the plateau, so the EITC is $8,231. It also ignored the 15%-of-earnings limit on the ACTC ($3,807.09), and its submitted $4,288 does not even match its own $8,688 sum." -us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model used a phase-out start of about $24,000 that is too low and phased the EITC down to $7,414. The MFJ three-child plateau extends beyond $27,880.57, so the full $8,231 applies. It also set the ACTC to the full $5,100 cap, ignoring the binding 15% × (earned income − $2,500) limit of $3,807.09." -us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model applied an EITC phase-out starting at about $24,000–$25,500 and settled on an arbitrary $5,475. A three-child MFJ household at $27,880.57 of earned income gets the full $8,231. It also computed the ACTC on gross $30,000 ($4,125) instead of on income net of the half-SE-tax deduction ($3,807.09)." -us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the CTC as entirely nonrefundable and set the ACTC to zero. With no tax liability and $27,880.57 of earned income, the refundable ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09. It also used the 2025 EITC maximum of $8,046 instead of the 2026 value of $8,231." +us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly worked out EITC $8,231 plus ACTC $3,807, for a total of about $12,038. It then rounded the final answer to $12,000 instead of reporting the computed amount." +us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It denied the EITC on the grounds that income was above a ~$50,162 phase-out endpoint, even though the $27,881 earned income is below that figure and is actually on the plateau for the full $8,231 maximum. It also worked out an ACTC of about $3,878 but then submitted an unexplained $2,500 and left out the EITC entirely." +us,scenario_109,federal_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated $30,000 of self-employment income as if it did not count as earned income, so it returned $0. Net SE earnings after the half-SE-tax deduction ($27,881) are earned income, which qualifies the household for the $8,231 EITC and a $3,807.09 refundable CTC." +us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own working reached EITC $8,231 + ACTC $3,807 = $12,038. It then discarded that result and submitted an unsupported $10,379." +us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 three-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231. It computed the ACTC on $27,705 without subtracting half of SE tax, then swapped in an unexplained $4,416 instead of the correct 15% × ($27,880.57 − $2,500) = $3,807.09." +us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It placed the household in the EITC phase-out and cut the credit to $5,035, but married-filing-jointly earned income of about $27,881 is below the phase-out start, so the full $8,231 applies. It also based the ACTC on gross $30,000 ($4,125) instead of earned income net of half SE tax ($3,807.09)." +us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It used a married-filing-jointly phase-out start of about $26,511–$27,174 (the actual MFJ threshold is above $30,000) and wrongly reduced the EITC to about $7,469. It also based the ACTC on gross $30,000 ($4,125) and then submitted $7,594, which drops $4,000 from its own $11,594 total. The correct figures are the full $8,231 EITC plus $3,807.09 ACTC on $27,880.57 of earned income." +us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It reduced the EITC to about $3,015, even though $27,881 of earned income is on the three-child plateau and earns the full $8,231. It also set the ACTC at the full $5,100 cap and ignored its own correct 15% × (earned income − $2,500) limit of $3,807." +us,scenario_109,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the ACTC 15% formula to gross $30,000 ($4,125) instead of earned income net of the half-SE-tax deduction ($27,880.57), which gives $3,807.09. It also used an EITC maximum of $8,215 instead of $8,231." +us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used an EITC of $8,026 instead of the 2026 three-child maximum of $8,231. It also used a $3,000 refundable CTC ($1,000 per child) instead of the post-OBBBA 15% × ($27,880.57 − $2,500) = $3,807.09." +us,scenario_109,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $3,807 is correct, but it overstated the 2026 three-child EITC maximum as $8,261 when the correct figure is $8,231." +us,scenario_109,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed the ACTC on gross $30,000 ($4,125) instead of earned income net of half SE tax ($27,880.57 → $3,807.09). It also overstated the 2026 EITC maximum as $8,290 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the expired pre-TCJA $1,000-per-child refundable CTC ($3,000). Under 2026 law the ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09, within a $1,700-per-child cap. It also overstated the EITC maximum as $8,290 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It cut the EITC to $3,588 as if the household were deep in the phase-out, but married-filing-jointly earned income of about $27,881 earns the full $8,231. It also understated the ACTC at $1,674 instead of 15% × ($27,880.57 − $2,500) = $3,807.09." +us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $1,000-per-child ACTC ($3,000) as if TCJA had expired, instead of the 2026 earned-income-based ACTC of $3,807.09. It also used an EITC maximum of $8,250 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It treated the ACTC as $1,000 per child ($3,000) under expired pre-TCJA rules, instead of 15% × ($27,880.57 − $2,500) = $3,807.09. It also used an EITC maximum of $8,240 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. The correct total is the full $8,231 three-child EITC plus a $3,807.09 ACTC, and its $7,502 is consistent with reporting roughly one partially phased-out credit and leaving out most of the other." +us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $1,000-per-child ACTC ($3,000) instead of the 2026 formula amount of 15% × ($27,880.57 − $2,500) = $3,807.09. It also used an EITC of $8,219 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $11,046 matches the 2025 EITC maximum ($8,046) plus a $1,000-per-child ACTC ($3,000). The correct inputs are the 2026 maximum of $8,231 and the earned-income-based ACTC of $3,807.09." +us,scenario_109,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It treated the refundable CTC as $1,000 per child ($3,000) instead of 15% × ($27,880.57 − $2,500) = $3,807.09. It also used an EITC maximum of $8,207 instead of $8,231." +us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used an EITC maximum of $8,025.75 instead of the 2026 three-child maximum of $8,231. It also computed the ACTC on net SE earnings of $27,705 without subtracting the $2,119.43 SE-tax deduction, which gives $3,780.75 instead of $3,807.09." +us,scenario_109,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted half of SE tax from the already-reduced $27,705 (92.35% of SE income) instead of from $30,000, which understated earned income at $25,586 rather than $27,880.57. That led to an ACTC of $3,463 instead of $3,807.09, and it also used an EITC of $8,042 instead of $8,231." +us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $13,400 matches the full EITC plus the full $1,700-per-child ACTC cap ($5,100). It ignored the earned-income limit that holds the ACTC to 15% × ($27,880.57 − $2,500) = $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 because no wages were listed, but self-employment income is earned income for both the EITC and the ACTC. With $27,880.57 of earned income the household receives the $8,231 EITC plus a $3,807.09 ACTC." +us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its EITC of $8,231 is correct, but it used the expired pre-TCJA $1,000-per-child ACTC ($3,000). Under 2026 law the ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09, within a $1,700-per-child cap." +us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 is correct, but it applied the 15% ACTC formula to gross $30,000 ($4,125). Earned income for self-employment is net of the deductible half of SE tax ($27,880.57), so the ACTC is $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 is correct, but it based the ACTC on $27,705 of net SE earnings (92.35% of income) instead of $30,000 minus half of SE tax ($27,880.57). That gives $3,781 instead of $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 is correct, but it computed the ACTC as 15% of gross $30,000 over $2,500 ($4,125). It did not first subtract the deductible half of SE tax, which brings earned income to $27,880.57 and the ACTC to $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 is correct, but it used gross $30,000 as earned income for the ACTC ($4,125). It should have used $30,000 minus the $2,119.43 half-SE-tax deduction, which gives $3,807.09." +us,scenario_109,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its ACTC of $3,807.09 is correct, but it overstated the 2026 three-child EITC maximum as $8,427 instead of $8,231." +us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It reported no refundable credits, overlooking that $30,000 of self-employment income is earned income. That earned income qualifies the family of three children for the full $8,231 EITC and a $3,807.09 ACTC." +us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA had expired, so it used a $1,000-per-child CTC, a $3,000 earnings threshold and a $3,000 ACTC. For 2026 the $2,500 threshold and $1,700-per-child refundable cap apply, which gives $3,807.09. It also used an EITC of $8,226 instead of $8,231." +us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules ($1,000 per child, $3,000 threshold) to get a $3,000 ACTC. Under the 2026 rules the ACTC is 15% × ($27,880.57 − $2,500) = $3,807.09. It also used an EITC maximum of $8,247 instead of $8,231." +us,scenario_109,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its EITC of $8,231 is correct, but it assumed TCJA had expired and capped the ACTC at $1,000 per child ($3,000). Under the 2026 rules the $2,500 threshold and $1,700-per-child cap apply, which gives 15% × $25,380.57 = $3,807.09." +us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used the expired-TCJA $1,000-per-child refundable CTC with a $3,000 threshold ($3,000 total) instead of the 2026 ACTC of $3,807.09. It also used an EITC of $8,245 instead of $8,231." +us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its ACTC of $3,807 is correct, but it overstated the 2026 three-child EITC maximum as about $8,271 instead of $8,231." +us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"It gave no reasoning. The correct total is the full $8,231 EITC (earned income of $27,880.57 is on the plateau) plus a $3,807.09 ACTC. Its $6,695 is about $5,343 short, which matches a wrongly phased-out EITC with most of the ACTC left out." +us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It said the phase-out starts around $29,600, which is above earned income, yet still cut the EITC to $3,588 instead of the full $8,231. It then dropped most of the ACTC from its final $4,288, despite earning a $3,807.09 refundable CTC under the 15%-over-$2,500 formula." +us,scenario_109,federal_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used the 2025 EITC maximum of $8,046 instead of the 2026 figure of $8,231. It also computed the ACTC on gross $30,000 ($4,125) instead of earned income net of half SE tax ($27,880.57), which gives $3,807.09." +us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It refunded the full $1,700-per-child cap ($5,100), ignoring the binding 15% × ($27,880.57 − $2,500) = $3,807.09 earned-income limit. It also phased out the EITC from a ~$24,000 start, which is not the joint-filer threshold, instead of awarding the full $8,231 to a joint filer with $27,881 of earned income." +us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It applied an understated MFJ phase-out start (~$24,000–$25,500) and cut the EITC to $5,475, but the household's $27,881 of earned income is below the joint phase-out start and earns the full $8,231. It also based the ACTC on gross $30,000 ($4,125) instead of $27,880.57 ($3,807.09)." +us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It treated the CTC as entirely nonrefundable and left out the $3,807.09 Additional CTC, which with zero tax liability equals 15% of earned income over $2,500. It also used the 2025 EITC maximum of $8,046 instead of $8,231." us,scenario_109,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted ineligibility without running either qualifying pathway: SNAP receipt of $8,020.55 confers categorical free-meal status by itself, and the household's $30,000 income is 78% of the 2026 five-person federal poverty guideline, far below the 130% free-meal threshold. Its bare 0 is consistent with demanding an explicit school-enrollment fact instead of deriving two K-12 students from the stated ages of 9 and 6, and Florida has no universal free-meal program that would make the answer turn on state policy." us,scenario_109,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated school-meal eligibility as an unlisted fact to be zeroed out under the 'treat unlisted household facts as false' instruction, so it never applied the SNAP categorical qualifier or the 130% poverty-guideline income test. Ages 9 and 6 establish two children in K-12, the household's $8,020.55 SNAP benefit is a standalone categorical pathway to free meals, and $30,000 is 78% of the five-person poverty guideline — each route independently yields the FREE tier and $2,261.92 of annual support." us,scenario_109,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated Florida as offering Medicaid expansion to low-income adults and therefore used low MAGI as sufficient for eligibility. Florida has no expansion pathway for this age-30 head, and the head fits no other categorical pathway, producing medicaid_category NONE and ineligibility." @@ -8778,48 +8889,49 @@ us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The mode us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"The model set up 30000 × 0.9235 × 0.153 correctly but evaluated it as 4234 when the product is 4238.87, understating the tax by $4.87 — its number implies a net-earnings base of $27,673 instead of $27,705. It then compounded the confusion with an irrelevant Additional Medicare Tax digression and a stray discarded figure of 2793 before committing to the miscomputed 4234." us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,other,False,"The model named the correct base (92.35% of $30,000) and the correct combined 15.3% Social Security and Medicare rate, but reported the product as $4,241 when $27,705 × 0.153 = $4,238.87, which rounds to $4,239. The $2.13 overstatement is a multiplication slip presented as a rounding, equivalent to using a net-earnings base of $27,719." us,scenario_109,self_employment_tax,minimax-m3,llm_error,other,False,"The model computed every component correctly — $27,705 net earnings, 12.4% Social Security of $3,435.42, 2.9% Medicare of $803.45 — and stated the total as approximately $4,238, then submitted $4,233, contradicting its own arithmetic by $5. Its own component figures sum to $4,238.87, so the submitted value is a transcription error rather than a rule or base mistake." -us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used a $225 standard deduction and a $1,158 maximum benefit instead of the FY2026 figures of $261 and $1,183. It then cut its own ~$625/month result to $3,768 a year by subtracting payroll/SE-tax adjustments. SNAP counts gross self-employment receipts minus business costs and never deducts SE tax, so the correct benefit is $1,183 − $522 = $661/month." -us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"Its deductions, net income ($1,739) and $1,183 maximum are all right. However, it did not round the 30% contribution ($521.70) up to the next whole dollar ($522), as SNAP requires. That left $661.30/month instead of $661, and $7,936 instead of $7,932." -us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It took half of SE tax off gross income, which SNAP does not do; SNAP counts the full $2,500/month of self-employment receipts. It also used a $1,316 maximum benefit instead of $1,183. Its final $244/month doesn't follow from its own numbers; the correct result is $1,183 − 30% of $1,739 net income (rounded up to $522) = $661." -us,scenario_109,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It denied SNAP without doing any calculation. The household's $2,500 monthly gross income is under the gross income limit, and its $900 in assets passes the asset test. Net income is $1,739, so the benefit is $1,183 − $522 = $661/month." -us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It used $1,613 as the five-person maximum benefit; the FY2026 figure is $1,183. It also used a $240 standard deduction instead of $261. With the correct figures, net income is $1,739 and the benefit is $661/month, not $1,085." -us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It cut self-employment income by 7.65% for SE tax, which SNAP does not do. It used a $217 standard deduction instead of $261. It also applied a shelter deduction even though the $800 rent is below half of adjusted income ($869.50). Its $11,436 answer ($953/month) doesn't match its own $717/month result; the correct benefit is $1,183 − $522 = $661." -us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"It claimed an excess shelter deduction from the $9,600 rent. None applies, because $800/month is below half of the $1,739 adjusted income. Its $424/month figure doesn't follow from the $1,183 maximum minus the $522 contribution, which gives $661/month." -us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took the FY2025 maximum benefit for five to be $967 and projected $1,017 for FY2026; the actual FY2026 figure is $1,183. It also used a $228 standard deduction instead of $261. Those two errors pushed the benefit down to $485/month instead of $661." -us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted six people; the household has five. It also cut SE income by 7.65%. It then concluded that income fails the limits even though its own figures (~$2,309 gross against a ~$4,590 limit) pass. The five-person household passes every test and gets $661/month." -us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its net income ($1,739) and $1,183 maximum are correct. However, it did not round the 30% contribution ($521.70) up to $522, so it got $661.30/month instead of $661 and $7,936 instead of $7,932." -us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $350 standard deduction instead of the FY2026 five-person figure of $261, which understated net income at $1,650 instead of $1,739. It also used a $1,185 maximum instead of $1,183. Together these inflated the benefit to $690/month instead of $661." -us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $232 standard deduction instead of $261 and a $1,184 maximum instead of $1,183. That overstated net income ($1,768 instead of $1,739) and gave $654/month instead of $661." -us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used outdated figures: a $244 standard deduction and a $1,158 maximum, instead of the FY2026 values of $261 and $1,183. That gave $631/month instead of $1,183 − $522 = $661." -us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It had the right $1,183 maximum but used a $250 standard deduction instead of the FY2026 five-person figure of $261. That raised net income to $1,750 and the contribution to $525, giving $658/month instead of $661." -us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its $1,750 net income reflects a ~$250 standard deduction instead of $261. Its $695/month benefit implies a maximum of about $1,220 instead of $1,183. The correct result is $1,183 − $522 = $661/month." -us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $765/month is $104 too high. It implies net income of about $1,393, as though a ~$346 shelter deduction applied. No excess shelter deduction exists, because $800 rent is below half of the $1,739 adjusted income, so the benefit is $661/month." -us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly subtracted an excess shelter deduction. That deduction is zero here: $800 rent minus half of $1,739 adjusted income ($869.50) is negative, and no utility costs are listed. Its $786/month should be $661." -us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a ~$250 standard deduction instead of $261, giving $1,750 net income. Its $665/month implies a maximum of about $1,190 instead of $1,183. The FY2026 figures give $661/month." -us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It answered $0 without doing any calculation. The five-person household with $2,500/month in earnings passes the gross, net and asset tests. Net income of $1,739 gives $1,183 − $522 = $661/month." -us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It treated the $800 rent as a deduction. SNAP only deducts shelter costs above half of adjusted income, and $800 is below $869.50, so the shelter deduction is zero. Its $752.50/month should be $661." -us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $732/month implies net income of about $1,503, which is $236 below the correct $1,739. That gap amounts to a deduction the household doesn't have; only the $500 earned income deduction and the $261 standard deduction apply, with no excess shelter. The correct benefit is $661/month." -us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $675/month is $14 too high, which reflects an inflated maximum or understated net income. The FY2026 figures ($261 standard deduction, $1,183 maximum) give net income of $1,739, a $522 contribution and $661/month." -us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"It used FY2025-era figures, a $236 standard deduction and a $1,155 maximum, instead of the FY2026 values of $261 and $1,183. It also skipped the whole-dollar round-up of the contribution. That produced $625.80/month instead of $661." -us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $258 standard deduction instead of $261 and a $1,180 maximum instead of $1,183. That gave ~$657/month, which it then rounded loosely to $7,880 a year, instead of $661/month and $7,932." -us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It claimed 'very low net income and high shelter costs,' but net income is $1,739 and the $800 rent produces no excess shelter deduction. Its $484.67/month is not derived from the $1,183 maximum minus the $522 contribution, which gives $661." -us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed no take-up and answered $0, even though the prompt says to assume program take-up. The household is eligible, with $1,739 net income and a benefit of $661/month." -us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $659/month matches using the FY2025 five-person standard deduction ($254) with the FY2026 $1,183 maximum. That gives $1,746 net income and a $524 contribution. The FY2026 standard deduction is $261, which gives $661/month." -us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $659/month matches using the outdated FY2025 $254 standard deduction instead of the FY2026 $261. That overstated net income at $1,746 and the contribution at $524, so the result is $2/month below the correct $661." -us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"Its parameters are correct ($261 standard deduction, $1,183 maximum, $1,739 net income). However, it did not round the $521.70 contribution up to $522, as SNAP requires, so it reported 12 × $661.30 = $7,935.60 instead of 12 × $661 = $7,932." -us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $8,045 works out to about $670.42/month, roughly $9 too high. That reflects an inflated maximum or an understated standard deduction. The FY2026 figures give $1,183 − $522 = $661/month." -us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"It got $1,739 net income and the $1,183 maximum right. However, it left the 30% contribution at $521.70 instead of rounding it up to $522, so it reported $661.30/month ($7,935.60) instead of $661 ($7,932)." -us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"It claimed income and assets exceed SNAP limits. In fact, $2,500/month is well under the five-person gross limit, and $900 in assets is far below the asset limit. The household qualifies for $661/month." -us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"It projected the five-person maximum at ~$1,213 instead of the actual FY2026 figure of $1,183. That inflated the benefit to $690/month instead of $661." -us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a ~$1,190 maximum instead of $1,183. Its other steps were right, so the error is the overstated maximum, which gave $668/month instead of $661." -us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"It used the FY2025 $254 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. It also applied the 30% contribution to annual income without the monthly round-up, which gave $7,610.40 instead of $7,932." -us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used the FY2025 $254 standard deduction instead of $261. It then made an arithmetic slip, giving $1,740 instead of $1,746. It also used a $1,189 maximum instead of $1,183, which pushed the benefit to $667/month instead of $661." -us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"It used a ~$250 standard deduction instead of $261 and a ~$1,190 maximum instead of $1,183. That gave $665/month instead of $661." -us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $300/month standard deduction instead of the FY2026 five-person $261 and a $1,220 maximum instead of $1,183. That understated net income ($1,700 instead of $1,739) and gave $710/month instead of $661." -us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"It treated rent as producing a capped shelter deduction and said net income was about $0. In fact, the $800 rent is below half of adjusted income, so there is no shelter deduction and net income is $1,739. It also used a $1,151 maximum instead of $1,183. Its $489/month should be $661." -us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"Its reasoning added a ~$400 Florida standard utility allowance, even though no utility expenses were listed, and used a $306 standard deduction instead of $261. That created a $356 shelter deduction that doesn't exist. Its submitted $7,920 ($660/month) also contradicts its own stated $9,400. With no shelter deduction and the contribution rounded up to $522, the benefit is $661/month." -us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It applied the earned income deduction at the gross income test and then assumed the maximum benefit, ignoring the 30% net income contribution. It also used a $962 maximum instead of $1,183. Net income of $1,739 reduces the benefit by $522, to $661/month." -us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It used a four-person maximum ($998); the household has five people, and the five-person maximum is $1,183. It also treated net income as zero. Net income is actually $1,739 after the $500 earned income deduction and $261 standard deduction, so the benefit is $661/month." +us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used outdated parameters: a ~$225 standard deduction and a $1,158 maximum allotment, instead of the FY2026 values of $261 and $1,183. Its own math gave about $626 a month. It then cut the answer to $3,768 ($314/month) by citing unspecified payroll/SE adjustments, which SNAP does not deduct from self-employment income with no listed costs." +us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"All of its parameters were correct ($1,739 net income, $1,183 maximum). However, it multiplied the unrounded $661.30 monthly benefit by 12 to get $7,936. SNAP rounds the 30% contribution up to $522, which makes the monthly benefit a whole $661, so the annual total is $7,932." +us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It subtracted half of the SE tax from countable income, which SNAP does not allow. It also used a made-up $1,316 maximum allotment instead of $1,183. Its final $244 a month does not even follow from its own numbers: $1,316 minus 30% of $1,830 is about $767." +us,scenario_109,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave $0 even though the household passes every test. Gross income of $2,500 a month is well under the limit, net income of $1,739 is under 100% FPL, and $900 in assets is under the limit. The correct benefit is $1,183 − $522 = $661 a month." +us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $1,613 maximum allotment for five people, far above the actual FY2026 figure of $1,183. It also used a $240 standard deduction instead of $261. That inflated the benefit to about $1,085 a month instead of $661." +us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It cut self-employment income by 7.65% for SE tax, which SNAP does not deduct, and used a $217 standard deduction instead of $261. It also applied a shelter deduction even though $800 rent is below half of adjusted income ($869.50). Its $11,436 result is about $953 a month, which does not match its own ~$717 monthly estimate." +us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"It claimed an excess shelter deduction, but $800 rent is below half of the $1,739 adjusted income, so none applies. Its $424 monthly benefit implies a contribution of about $759 against the $1,183 maximum. That would mean net income above the $2,500 gross income, so the deductions were not actually applied. The correct net income is $1,739, which gives $661 a month." +us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It guessed a $1,017 maximum allotment and a $228 standard deduction. The FY2026 values for five people are $1,183 and $261. That produced $485 a month instead of $661." +us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted six people instead of five and reduced SE income by an SE-tax factor. It then said net income was over the limit, which contradicts its own figures showing income well below both limits. The correct result is a $661 monthly benefit." +us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used the correct $1,739 net income and $1,183 maximum. However, it annualized the unrounded $661.30 to get $7,936. SNAP rounds the contribution up to $522 and pays a whole $661 a month, which is $7,932 a year." +us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $350 standard deduction instead of the FY2026 five-person value of $261, and a $1,185 maximum instead of $1,183. That understated net income ($1,650 vs $1,739) and overstated the benefit ($690 vs $661 a month)." +us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $232 standard deduction instead of $261, which overstated net income ($1,768 vs $1,739) and the contribution ($530 vs $522). It also used a $1,184 maximum instead of $1,183. The result was $654 a month instead of $661." +us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $244 standard deduction and a $1,158 maximum allotment instead of the FY2026 values of $261 and $1,183. That gave $631 a month instead of $661." +us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its maximum allotment ($1,183) was correct, but it used a $250 standard deduction instead of $261. That made net income $1,750 and the contribution $525, giving $658 a month instead of $661." +us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used net income of $1,750, which reflects a $250 standard deduction instead of $261. Its $695 monthly benefit also implies a maximum allotment of about $1,220 instead of $1,183, so the benefit was overstated." +us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation. Its $9,180 ($765 a month) implies either a maximum allotment well above $1,183 or net income far below the correct $1,739. With the 20% earned income deduction and $261 standard deduction, and no shelter deduction, the benefit is $661 a month." +us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied an excess shelter deduction, but $800 rent is below half of the $1,739 adjusted income ($869.50), so the deduction is zero. Its $786 a month implies net income of about $1,323 instead of $1,739." +us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $250 standard deduction instead of $261, giving net income of $1,750. Its $665 a month also implies a maximum allotment of about $1,190 instead of $1,183. Both errors push the benefit above the correct $661." +us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave $0 with no computation, even though the household passes the gross income, net income and asset tests. The correct result is $1,183 minus a $522 contribution, or $661 a month." +us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It treated the $800 rent as producing a deduction. The excess shelter deduction is only the amount of shelter costs above half of adjusted income ($869.50), which here is zero. Its $752.50 a month overstates the correct $661." +us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no computation, and $8,784 ($732 a month) is not a whole-dollar result of the FY2026 rules. With $1,739 net income and a $1,183 maximum, the benefit is $661 a month. Its number implies an inflated maximum allotment or understated net income." +us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It gave no computation. Its $675 a month exceeds the correct $1,183 − $522 = $661, so it used too high a maximum allotment or too large a deduction instead of the $261 standard deduction and $500 earned income deduction." +us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"It used FY2025-level values: a $236 standard deduction and a $1,155 maximum allotment, instead of the FY2026 values of $261 and $1,183. It also annualized the unrounded $625.80. The result was $7,509.60 instead of $7,932." +us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $258 standard deduction and a $1,180 maximum allotment instead of $261 and $1,183. That gave about $657 a month instead of $661." +us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It described the household as having very low net income and high shelter costs. In fact net income is $1,739 and rent is below half of adjusted income, so there is no shelter deduction. Its $5,816 ($485 a month) does not follow from the $1,183 maximum minus the $522 contribution, which is $661." +us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed no take-up and returned $0, but the prompt says to assume program take-up. The household is eligible for $661 a month." +us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $659 a month is $2 below the correct $661. It used a slightly wrong standard deduction or maximum allotment instead of the FY2026 values of $261 and $1,183, which give a $522 contribution." +us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $659 a month is $2 below the correct $661. That reflects a parameter slightly off from the FY2026 $261 standard deduction and $1,183 maximum allotment, which give net income of $1,739 and a contribution of $522." +us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"Its parameters were correct, but it computed 12 × $661.30 = $7,935.60. SNAP rounds the contribution up to $522, which makes the monthly benefit a whole $661, so the annual total is $7,932." +us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $8,045 equals about $670.42 a month, which is neither a whole-dollar allotment nor consistent with the FY2026 rules. A $261 standard deduction and $1,183 maximum give $661 a month." +us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"It derived $661.30 a month correctly but did not round to a whole dollar before annualizing. SNAP pays $661 a month after rounding the contribution up to $522, which is $7,932 a year, not $7,935.60." +us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"It said income and assets exceed SNAP limits, but they do not. Gross income of $2,500 a month is far below 130% FPL (about $4,079 for five), net income of $1,739 is below 100% FPL, and $900 in assets is under the limit. The correct benefit is $661 a month." +us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"It projected a $1,213 maximum allotment instead of the actual FY2026 $1,183, and a standard deduction of about $256 instead of $261. That overstated the benefit at $690 a month instead of $661." +us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a maximum allotment of about $1,190 instead of $1,183. Its net income of about $1,740 was close to the correct $1,739, so the benefit came out at $668 a month instead of $661." +us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"It used a $254 standard deduction and a $1,158 maximum allotment instead of the FY2026 values of $261 and $1,183. It also computed the benefit on an annual basis without whole-dollar monthly rounding, giving $7,610.40." +us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $1,189 maximum allotment instead of $1,183 and a $254 standard deduction instead of $261. It also made a subtraction error: $2,000 − $254 is $1,746, not $1,740. The result was $667 a month instead of $661." +us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"It used a ~$250 standard deduction instead of $261 and a ~$1,190 maximum allotment instead of $1,183. That gave $665 a month instead of $661." +us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It projected a $300 monthly standard deduction instead of $261 and a $1,220 maximum allotment instead of $1,183. That understated net income ($1,700) and overstated the benefit ($710 vs $661 a month)." +us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"It applied an excess shelter deduction and treated net income as about $0. In fact rent is below half of adjusted income, so there is no shelter deduction and net income is $1,739. It also used a $1,151 maximum instead of $1,183, and its $489 monthly result does not follow from its own premises." +us,scenario_109,snap,muse-wonder,llm_error,thresholds_rates,False,"It used a $1,158 maximum allotment instead of the FY2026 $1,183. Its $1,704 net income implies a standard deduction of about $296 instead of $261. The result was $647 a month instead of $661." +us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"Its reasoning added a ~$400 Florida standard utility allowance even though no utility costs are listed, and used a $306 standard deduction instead of $261. That created a shelter deduction that does not exist. The $7,920 it submitted (and the $9,400 in its reasoning) both miss $661 a month, which is $7,932 a year." +us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It treated the household as getting a near-maximum benefit with a $962 cap. The FY2026 maximum for five is $1,183, and net income of $1,739 requires a $522 contribution. The correct benefit is $661 a month, not $962." +us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It used a four-person household instead of five and treated countable net income as zero. Net income after the $500 earned income and $261 standard deductions is $1,739, which reduces the $1,183 five-person maximum to $661 a month." us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a federal minimum Medicaid pathway for any household member with no individual income and few assets. Florida requires this nondisabled adult to satisfy a covered category, and Medicaid MAGI eligibility uses household income rather than treating the spouse’s unlisted personal income as an isolated $0 eligibility base." us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly calculated that household income exceeds Florida’s parent/caretaker limit and explicitly concluded that the spouse is not eligible, but then submitted value 1. No alternative Medicaid category applies to reverse its own eligibility analysis." us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly converted the presence of an infant into pregnancy or postpartum eligibility. An age-zero child does not establish that the spouse is pregnant, postpartum, or the child’s mother, and caretaker status alone fails because household income exceeds Florida’s parent/caretaker threshold." @@ -8833,116 +8945,119 @@ us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligib us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. Because postpartum status is not listed, it is false under the prompt, leaving the spouse outside every adult WIC category." us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred postpartum or breastfeeding status from the spouse's age and the infant's presence. Neither fact establishes the spouse's own qualifying status, and the prompt explicitly makes those unlisted statuses false." us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum status or current breastfeeding from the existence of a zero-year-old child. Those personal statuses were unlisted and therefore false, so the spouse fails categorical eligibility before the income threshold affects the result." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached $25,840.76, but it submitted $21,120, a number with no support in that reasoning. Its derivation also left out the 2026 non-itemizer charitable cash deduction ($387), which cuts about $93 of 24%-bracket tax." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used a $15,000 standard deduction instead of OBBBA's $16,100. It deducted the $198 traditional IRA contribution, which is fully phased out for an active plan participant with about $170k of MAGI. It said charitable gifts cannot reduce tax, missing the $387 non-itemizer cash deduction. It then subtracted an invented $1,300 of nonrefundable credits from its $27,137." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed the tax and submitted a $0 placeholder, even though its own reasoning said the liability is substantial. The correct steps give $19,615 of ordinary-rate tax on $112,571.70 plus $6,133 of 15% tax on $40,884 of preferential income." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used guessed brackets (12,150/49,400/105,450) instead of the OBBBA 2026 single brackets (12,400/50,400/105,700), which overstated ordinary tax. It also skipped the $387 non-itemizer charitable cash deduction, so its taxable income was $153,843 instead of $153,455.47." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It allowed the $198 traditional IRA deduction, which is fully phased out for an active 401(k) participant with $169,943 of MAGI. It omitted the $387 non-itemizer charitable cash deduction and set the top of the 22% bracket at $107,500 instead of $105,700. Its total landed near the reference only because these errors partly cancelled." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It added the $6,133 preferential-rate tax on top of a total that already included it, ending at $30,150 even though its components (about $20,000 ordinary plus $6,133) add to about $26,150. It also allowed the phased-out $198 IRA deduction and missed the $387 non-itemizer charitable deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It made up a $10,400 qualified overtime deduction from the hourly rate and weekly hours, but no overtime premium was given and unlisted amounts are 0. It also allowed the phased-out $198 IRA deduction and missed the $387 non-itemizer charitable deduction. Together these understated taxable income by about $10,000." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and its own inflation-guessed brackets instead of OBBBA's 2026 figures ($16,100 deduction; brackets at 12,400/50,400/105,700). It also left out the $387 non-itemizer charitable cash deduction, which together overstated ordinary-rate tax." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It itemized using a $5,331 deduction for unreimbursed employee business expenses above a 2% floor, but OBBBA permanently eliminated that deduction. It applied 2025 brackets instead of 2026 ones. It then cut its own $26,044.70 result to $20,200 for no stated reason." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $16,100 standard deduction, the 2026 brackets and the 15% preferential tax right, but missed the 2026 non-itemizer charitable cash deduction. Deducting the $387 of cash gifts lowers ordinary taxable income to $112,571.70 and the tax by about $93." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the old $15,000 standard deduction instead of OBBBA's $16,100 for 2026 and deducted the $198 IRA contribution, which phases out at this income. It also missed the $387 non-itemizer charitable deduction and overstated ordinary tax at $20,174." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a $4,850 personal exemption, the 2% miscellaneous deduction and old brackets that run up to 28%. It also subtracted the $6,889 of employer-paid premiums from wages, even though wages are given gross and only the 401(k) deferral reduces them. Its $32,490 ordinary tax on about $101k is also miscalculated." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules (a $5,450 personal exemption, an $8,650 standard deduction, and itemizing that includes miscellaneous deductions), but OBBBA made the TCJA structure permanent. It also wrongly took the $6,889 of employer premiums out of wages and allowed the phased-out $198 IRA deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the correct $16,100 standard deduction and 2026 brackets, but missed the non-itemizer charitable cash deduction ($387) that OBBBA added starting in 2026. This left taxable income $387 too high and the tax about $93 too high." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, taking a $5,050 personal exemption and a $5,331 deduction for 2%-floor employee business expenses, and taxing ordinary income at pre-TCJA rates. Under OBBBA the correct calculation is the $16,100 standard deduction plus the $387 charitable deduction, with the 10/12/22/24% brackets." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It left out the $6,323 of estate income. Its $19,688 is about the ordinary-rate tax alone ($19,615), with no 15% tax on the $40,884 of qualified dividends and LTCG, which adds $6,132.56." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA had expired, using personal exemptions and itemized deductions, even though OBBBA made the TCJA rate and deduction structure permanent. It also subtracted pre-tax health premiums from wages, even though only the $4,245 traditional 401(k) deferral reduces reported wages." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied rules from a TCJA expiration that did not happen: a $5,050 personal exemption, employee business expenses above a 2% AGI floor, and pre-TCJA brackets that gave $20,928 of ordinary tax. The OBBBA approach is the $16,100 standard deduction plus the $387 non-itemizer charitable deduction, with 2026 brackets." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $19,679 is about the ordinary-rate tax alone ($19,615 on $112,571.70) and leaves out the $6,132.56 of 15% tax on the $40,884 of qualified dividends and long-term gains." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took the $6,889 of employer-sponsored premiums out of wages, giving $163,054 of AGI, but only the $4,245 traditional 401(k) deferral reduces wages, so AGI is $169,942.94. That understated taxable income and the ordinary-rate tax." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It began from $163,054 of AGI after subtracting $6,889 of employer premiums from wages, which the facts do not support. Only the 401(k) deferral reduces wages, so the correct AGI is $169,942.94." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of OBBBA's $16,100 for 2026, used off-cycle brackets, and left out the $387 non-itemizer charitable deduction. This overstated ordinary taxable income at $113,709 versus $112,571.70." -us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied 2025 brackets (11,925/48,475/103,350) and a guessed $15,400 standard deduction instead of 2026's $16,100 deduction and 12,400/50,400/105,700 brackets. It also left out the $387 non-itemizer charitable cash deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It placed the $40,884 of qualified dividends and LTCG in the 0% capital gains bracket. Those gains sit on top of $112,572 of ordinary income, far above the roughly $49,450 0% threshold, so all of them are taxed at 15% ($6,132.56). It also missed the $387 charitable deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its own reasoning said tax before refundable credits is positive, but it submitted $0. The correct result is $19,615 of ordinary-rate tax plus $6,133 of preferential-rate tax." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized deductions such as mortgage interest, real estate taxes and medical expenses, but those total less than the $16,100 standard deduction. Its $16,811 is about $8,900 below the correct total, which is $19,615 of ordinary tax on $112,571.70 plus $6,133 at 15% on $40,884 of preferential income." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It correctly used $169,943 of AGI, the $16,100 standard deduction and 15% preferential rates, but missed the 2026 non-itemizer charitable cash deduction. Subtracting the $387 of cash gifts reduces ordinary taxable income to $112,571.70 and the tax by about $93." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly applied the $387 non-itemizer charitable deduction, but its total is about $137 below $25,747.77. That gap comes from an arithmetic error in the 24%-bracket tax on ordinary taxable income of $112,571.70, which should be $19,615.21." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used only the $16,100 standard deduction and missed the separate non-itemizer charitable cash deduction for 2026. The $387 of cash donations lowers taxable income to $153,455.47 and the tax to $25,747.77." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It took only the $16,100 standard deduction and left out the $387 non-itemizer charitable cash deduction that OBBBA added for 2026. That left ordinary taxable income at $112,959 instead of $112,571.70." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It took $6,889 of employer premiums out of AGI, giving $163,054 instead of $169,942.94. It also claimed a $9,000 overtime deduction, but no overtime premium was given and unlisted amounts are 0. Both errors understated taxable income." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It missed the 2026 non-itemizer charitable cash deduction. Subtracting the $387 of cash gifts on top of the $16,100 standard deduction lowers ordinary taxable income to $112,571.70 and the tax to $25,747.77." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It made up an $8,400 qualified overtime deduction from the hours worked, but no overtime premium was given and unlisted amounts are 0. It also left out the $387 non-itemizer charitable deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $6,889 of employer-sponsored premiums from wages as if they were salary reduction, giving $163,054 of AGI. Only the $4,245 traditional 401(k) deferral reduces wages, so AGI is $169,942.94 and ordinary taxable income is $112,571.70." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It assumed itemized deductions and 'standard nonrefundable credits' that do not exist for this filer. The itemizable items total less than the $16,100 standard deduction, and no nonrefundable credits apply, so the full $25,747.77 stands." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied rules from a TCJA expiration that did not happen: pre-TCJA brackets, a personal exemption, and miscellaneous deductions above a 2% floor. Under OBBBA, 2026 has the $16,100 standard deduction, the $387 non-itemizer charitable deduction and the 10/12/22/24% brackets." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA had expired, using the 10/15/25% brackets, a $5,300 personal exemption and 2%-floor employee expenses. OBBBA made the TCJA structure permanent, so the correct calculation is the $16,100 standard deduction plus the $387 charitable deduction with 2026 brackets." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used rules from a TCJA expiration that did not happen: a $5,450 personal exemption, 2%-floor employee expenses, and the 10/15/25% brackets. Under OBBBA the correct calculation is the $16,100 standard deduction plus the $387 non-itemizer charitable deduction with 2026 brackets." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying a personal exemption, a 2%-floor miscellaneous deduction and the 10/15/25% brackets. It also deducted the $198 IRA contribution, which phases out for an active plan participant at this MAGI." -us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized using $5,331 of miscellaneous employee expenses above a 2% floor and an estimated Ohio income tax. OBBBA permanently eliminated miscellaneous itemized deductions, so the remaining itemized items fall short of the $16,100 standard deduction. The correct path is the standard deduction plus the $387 non-itemizer charitable deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no usable value and no reasoning for federal_income_tax_before_refundable_credits. -us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It claimed an $8,425.50 qualified overtime deduction based on an estimated overtime premium, but no overtime premium was given and unlisted amounts are 0. It also deducted the $198 traditional IRA contribution, which is fully phased out at this MAGI for an active participant." -us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It claimed a $0 PolicyEngine result that it never computed. A single filer with $153,455.47 of taxable income owes $19,615 of ordinary-rate tax plus $6,133 of 15% tax on preferential income." -us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $26,973 but submitted $43,879.20, which has no support in its own reasoning. It also never subtracted the $4,245 traditional 401(k) deferral from wages, used a $15,750 standard deduction, and left out the $387 charitable deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $113,617 AGI drops about $56,000 of income from the correct $169,942.94. It made up a QBI deduction even though there is no qualified business income, and it subtracted $194.77 of nonrefundable credits that do not apply." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working reached $25,841, using the right AGI, standard deduction and brackets, but it submitted $21,120, a figure with no support in its reasoning. Its computed figure also leaves out the 2026 non-itemizer charitable deduction for the $387 of cash gifts, which brings taxable income down to $153,455 and the tax to $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of the 2026 figure of $16,100, deducted the $198 traditional IRA even though the deduction is phased out for an active plan participant at this income, and put ordinary tax at $21,009 using outdated brackets. It then subtracted an invented $1,300 of nonrefundable credits, although none apply, and it never took the $387 non-itemizer charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It submitted 0 as a placeholder without computing anything, even though its own reasoning said the tax is substantial. The correct calculation is AGI $169,943, minus the $16,100 standard deduction and the $387 charitable deduction, giving $19,615 of ordinary tax plus $6,133 at 15%, or $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used guessed bracket thresholds ($12,150/$49,400/$105,450) rather than the 2026 single thresholds of $12,400/$50,400/$105,700, which pushed ordinary tax up to $19,818. It also left out the $387 non-itemizer cash charitable deduction, which brings ordinary taxable income down to $112,572." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It allowed the $198 traditional IRA deduction even though the active-participant phaseout eliminates it at about $170k of MAGI. It also set the top of the 22% bracket at about $107,500 instead of $105,700 and never applied the $387 non-itemizer charitable deduction. These errors happened to offset and landed near $25,758." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It counted the preferential-rate tax twice, adding $6,133 on top of a $26,150 figure that already included it, for $30,150. It also took the phased-out $198 IRA deduction and missed the $387 non-itemizer charitable deduction. The correct total is $19,615 ordinary tax plus $6,133 at 15%." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It made up a $10,400 qualified-overtime deduction from the hourly rate and weekly hours, although no overtime compensation is reported. It also deducted the phased-out $198 IRA contribution and left out the $387 non-itemizer charitable deduction. Together these brought taxable income down to $143,245 instead of $153,455." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $15,750, the 2025 standard deduction, instead of $16,100 for 2026, and inflated the 2025 brackets itself instead of using $12,400/$50,400/$105,700. It also never applied the $387 non-itemizer cash charitable deduction, so ordinary tax came out at $19,933 rather than $19,615." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA would revert and itemized $5,331 of unreimbursed employee expenses above the 2% floor, a deduction that has been permanently eliminated. It also used 2025 brackets, and then replaced its own $26,045 result with an unsupported $20,200. The correct calculation uses the $16,100 standard deduction plus the $387 cash charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $16,100 standard deduction and the 2026 brackets right, but missed the 2026 non-itemizer deduction for the $387 of cash charitable gifts. Ordinary income was therefore taxed on $112,959 instead of $112,572, leaving its answer about $93 too high." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of $16,100 for 2026 and deducted the $198 IRA contribution, which is phased out for an active participant. It also missed the $387 non-itemizer charitable deduction and applied outdated bracket thresholds, so its ordinary tax came out at $20,174." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules: a $4,850 personal exemption, a $6,800 standard deduction, 2%-floor employee expenses and 10–28% brackets. It also excluded ESI premiums from wages, and its ordinary tax of $32,490 doesn't follow from its own taxable income. Under 2026 law the correct calculation uses the $16,100 standard deduction plus the $387 charitable deduction on AGI of $169,943." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It took $6,889 of ESI premiums out of wages and deducted the phased-out $198 IRA contribution. It also applied a $5,450 personal exemption, an $8,650 standard deduction and itemized deductions as if the TCJA had expired. The correct calculation keeps wages at $95,755 and uses the $16,100 standard deduction plus the $387 cash charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left out the 2026 non-itemizer deduction for the $387 of cash charitable gifts. Taxable income should be $153,455 rather than $153,843, which brings ordinary tax down to $19,615 instead of $19,708." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA would sunset, so it applied a $5,050 personal exemption and itemized $5,331 of unreimbursed employee expenses above a 2% floor. Neither exists under the permanent 2026 rules, which use the $16,100 standard deduction plus the $387 non-itemizer charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its $19,688 matches only the ordinary-rate tax, which is about $19,600–$19,700. It left out the $6,133 of 15% tax on the $40,884 of qualified dividends and long-term gains, and it did not apply the $387 charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It took pre-tax health premiums out of wages and assumed the TCJA had sunset, applying personal exemptions and itemized deductions. Under 2026 law the correct calculation uses wages of $95,755, the $16,100 standard deduction and the $387 non-itemizer charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA rules: a $5,050 personal exemption and an itemized total that includes 2%-floor unreimbursed employee expenses. Both were permanently eliminated, so the correct deductions are the $16,100 standard deduction plus the $387 non-itemizer cash charitable deduction, giving $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Its $19,679 is roughly just the ordinary-rate tax on about $112.6k of ordinary taxable income. It never added the $6,133 of 15% tax on the $40,884 of qualified dividends and long-term capital gains." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took the $6,889 of employer-sponsored insurance premiums out of wages, which gave an AGI of $163,054. The 401(k) deferral is the only reduction to wages, so wages are $95,755 and AGI is $169,943. That lower AGI pulled its tax down to $22,415." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $163,054 by excluding the $6,889 of ESI premiums from wages, but only the $4,245 traditional 401(k) reduces wages, so AGI is $169,943. That understated taxable income and gave $22,765." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of $16,100 for 2026 and missed the $387 non-itemizer charitable deduction. As a result it taxed $113,709 of ordinary income instead of $112,572." +us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a guessed $15,400 standard deduction and 2025 bracket thresholds ($11,925/$48,475/$103,350) instead of the 2026 figures ($16,100; $12,400/$50,400/$105,700). It also never took the $387 non-itemizer cash charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It put the $40,884 of qualified dividends and LTCG in the 0% bracket. Because they stack on top of $112,572 of ordinary income, well above the 0% threshold of about $49,450, all of that income is taxed at 15% ($6,133). It also missed the $387 non-itemizer charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"Its reasoning says the tax stays positive, yet it submitted 0 without doing any calculation. The correct calculation gives $19,615 of ordinary tax plus $6,133 of 15% preferential tax, or $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $16,811 is about $8,900 too low. That is consistent with applying medical and itemized deductions that don't beat the standard deduction and with not taxing the $40,884 of preferential income at 15%. The correct calculation uses the $16,100 standard deduction plus the $387 cash charitable deduction, giving $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the standard deduction but left out the 2026 non-itemizer deduction for the $387 of cash gifts. Taxable income should be $153,455 rather than $153,843, and the tax $25,748 rather than $25,841." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It applied the $387 non-itemizer charitable deduction correctly, but its $25,611 is $137 short, the equivalent of about $570 of extra deductions or bracket error at the 24% rate. With AGI of $169,943, the $16,100 standard deduction and the 2026 brackets, ordinary tax is $19,615.21 and preferential tax is $6,132.56." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It missed the 2026 non-itemizer charitable deduction for the $387 of cash donations, so it taxed $388 too much ordinary income at 24%. That left its answer about $93 above $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied only the $16,100 standard deduction and skipped the extra 2026 non-itemizer deduction for the $387 of cash charitable gifts. Its $25,840.76 is therefore $92.99 too high." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It took $6,889 of ESI premiums out of wages, giving an AGI of $163,054 instead of $169,943. It also claimed an estimated $9,000 qualified-overtime deduction even though no overtime compensation is reported. Only the $16,100 standard deduction and the $387 charitable deduction apply." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left out the 2026 non-itemizer deduction for the $387 of cash gifts, so taxable income stayed at $153,843 instead of $153,455. That overstated the 24%-bracket tax by about $93." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It deducted $8,400 of qualified overtime even though no overtime compensation is reported, and it left out the $387 non-itemizer charitable deduction. The correct taxable income is $153,455, not $145,443." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It took the $6,889 of employer-plan premiums out of wages, giving an AGI of $163,054. Only the $4,245 traditional 401(k) reduces wages, so AGI is $169,943 and ordinary taxable income is $112,572 rather than $105,683." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It guessed about $24.5k without doing a calculation, and assumed nonrefundable credits that don't apply to a single filer with no dependents. The correct calculation gives $19,615 of ordinary tax plus $6,133 of 15% preferential tax, or $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied restored pre-TCJA 2026 rules, including a personal exemption of about $5,300, 2%-floor employee expenses and the old 10/15/25% brackets. The TCJA structure is permanent, so the correct calculation uses the $16,100 standard deduction, the $387 non-itemizer charitable deduction and the 10/12/22/24% brackets." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA would sunset, applying a $5,300 personal exemption, $5,331 of 2%-floor employee expenses and the 10/15/25% brackets. Under permanent 2026 law the deductions are the $16,100 standard deduction plus the $387 cash charitable deduction, taxed at the 10/12/22/24% rates." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration rules: a $5,450 personal exemption, itemized deductions that include $5,331 of 2%-floor employee expenses, and the 10/15/25% brackets. The correct calculation uses the permanent $16,100 standard deduction plus the $387 non-itemizer charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted the phased-out $198 IRA contribution and applied pre-TCJA rules: a $5,387 personal exemption, 2%-floor miscellaneous deductions and the 10/15/25% brackets. The correct calculation uses AGI of $169,943, the $16,100 standard deduction and the $387 non-itemizer charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized $21,146 by adding $5,331 of 2%-floor unreimbursed employee expenses, which are permanently non-deductible, plus an estimated Ohio income tax. Without those, the filer takes the $16,100 standard deduction plus the $387 non-itemizer charitable deduction, giving taxable income of $153,455 rather than $148,797." +us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer or explanation for this variable, so there was nothing to score. The correct value is $19,615.21 of ordinary tax plus $6,132.56 of capital gains tax, or $25,747.77." +us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It claimed an $8,425.50 phased overtime deduction even though no overtime compensation is reported, and deducted the $198 IRA contribution, which is phased out for an active participant. Those errors cut taxable income to $144,833 instead of $153,455, although it applied the $387 charitable deduction correctly." +us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It submitted 0 on the strength of a claimed PolicyEngine result it never actually computed. A single filer with AGI of $169,943 and no credits owes $19,615 of ordinary tax plus $6,133 of 15% preferential tax, or $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It took the employer health premiums out of wages, giving an AGI of $163,054. Only the $4,245 traditional 401(k) reduces wages, so AGI is $169,943, and with the $16,100 standard deduction plus the $387 charitable deduction the tax is $25,748." +us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its own working reached $26,973, but it submitted an unrelated $43,879.20. That working was also off: it left out the $4,245 401(k) deferral and the $18 of STCG, used a $15,750 standard deduction and rough brackets, and omitted the $387 charitable deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reported AGI of $113,617, dropping about $56k of the interest, dividend and capital gains income, so the real AGI is $169,943. It also invented a QBI deduction and $194.77 of nonrefundable credits, although the filer has no qualified business income and no credits." us,scenario_110,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_110,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its own reasoning says employee Social Security and Medicare tax on $100,000 of wages is about $7,650, yet it submitted 0. Nothing in the facts zeroes out FICA on $100,000 of W-2 wages, so the submitted value contradicts its own correct calculation." -us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It treated the $6,889 employer-sponsored insurance premium as a pre-tax cafeteria-plan reduction and cut FICA wages to $93,111. The facts list no Section 125 salary reduction, so the 7.65% FICA rate applies to the full $100,000 of gross wages, which gives $7,650." -us,scenario_110,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It subtracted the $6,889 employer-sponsored insurance premium from wages to get FICA wages of $93,111. The payroll tax base is the full $100,000 of gross wages: $6,200 Social Security plus $1,450 Medicare, for $7,650." -us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut FICA wages by the $6,889 employer-sponsored health premium and applied 7.65% to $93,111. The payroll tax base is the full $100,000 of listed gross wages, which gives $7,650." -us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $6,889 health premium was paid pre-tax and lowered FICA wages to $93,111. Social Security and Medicare tax apply to the full $100,000 of gross wages ($6,200 + $1,450 = $7,650)." -us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It excluded the $6,889 employer-sponsored premium from FICA wages, giving a base of $93,111. The correct base is the full $100,000 of gross wages, so employee FICA is $7,650." -us,scenario_110,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It subtracted $6,889 of pre-tax health insurance to get FICA wages of $93,111. The 7.65% employee FICA rate applies to the full $100,000 of gross wages, which gives $7,650." -us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It stated the correct rule (6.2% up to the wage base plus 1.45% Medicare on wages only), but its $8,210 equals 7.65% of about $107,320. That means it inflated the wage base above the $100,000 of listed wages. Applying its own rule to $100,000 gives $7,650." -us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $4,245 traditional 401(k) deferral and applied FICA to $95,755. Under IRC §3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base is the full $100,000 and the tax is $7,650." -us,scenario_110,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It reduced FICA wages by $6,889 of 'pretax' employer-sponsored insurance premiums to $93,111. The facts give no salary-reduction election, so 7.65% applies to the full $100,000 of gross wages, which gives $7,650." -us,scenario_110,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It correctly kept the 401(k) deferral in FICA wages but subtracted the $6,889 employer-plan premium, leaving a $93,111 base. The payroll tax base is the full $100,000 of gross wages, so employee FICA is $7,650." -us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It excluded $6,889 of employer-sponsored health premiums from FICA wages, which left $93,111. The correct base is the full $100,000 of gross wages, so Social Security is $6,200, Medicare is $1,450, and the total is $7,650." -us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for payroll_tax. The correct output is $7,650 of employee FICA on $100,000 of wages." -us,scenario_110,payroll_tax,minimax-m3,llm_error,other,False,"It claimed without basis that PolicyEngine returns 0 and never computed anything. $100,000 of W-2 wages carries $6,200 of employee Social Security tax and $1,450 of Medicare tax, for a total of $7,650." -us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly totals $6,200 Social Security plus $1,450 Medicare as $7,650, but it submitted $9,351.90. That value matches no step in its own calculation." -us,scenario_110,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It added $252.55 of Additional Medicare Tax, which is 0.9% on about $28,061 above $200,000, so it counted non-wage investment income toward the threshold. The 0.9% Additional Medicare Tax applies only to wages and self-employment income over $200,000, and wages here are $100,000. The correct total is $7,650." +us,scenario_110,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its own reasoning says employee Social Security and Medicare on $100,000 of wages come to about $7,650, but it submitted 0 anyway. It skipped the final computation and returned a placeholder zero instead of the $6,200 + $1,450 it had already identified." +us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It treated the $6,889 employer-sponsored insurance premium as a pre-tax cafeteria-plan deduction and cut FICA wages to $93,111. The household facts list no salary reduction, so FICA applies to the full $100,000 of gross wages, giving $6,200 + $1,450 = $7,650." +us,scenario_110,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It subtracted the $6,889 employer-sponsored insurance premium from gross wages, which gave $93,111 of FICA wages. Payroll tax applies to the full $100,000 of stated gross wages, so the correct figure is 7.65% × $100,000 = $7,650." +us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It defined FICA wages as gross wages minus the $6,889 employer-sponsored insurance premium. The stated gross wages of $100,000 are not reduced by that premium, so the 7.65% employee rate yields $7,650, not $7,123." +us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $6,889 of ESI premiums were paid pre-tax and lowered FICA taxable wages to $93,111. Social Security (6.2%) and Medicare (1.45%) apply to the full $100,000, giving $6,200 + $1,450 = $7,650." +us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It excluded the $6,889 employer-sponsored health premium from FICA wages, although nothing in the facts says it was a salary reduction. Employee FICA on the full $100,000 of gross wages is $7,650." +us,scenario_110,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It took $6,889 off gross wages as pre-tax health insurance before applying FICA. The payroll-tax base is the full $100,000 of gross wages, which yields $7,650 at the 7.65% combined employee rate." +us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Its stated rule (6.2% Social Security plus 1.45% Medicare on wages, with no state payroll tax) applied to $100,000 gives exactly $7,650. Its $8,210 matches applying 7.65% to about $107,320, a wage base above the stated gross wages, so it applied the rates to an inflated wage figure." +us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $4,245 traditional 401(k) deferral from FICA wages, which gave $95,755. Elective 401(k) deferrals stay subject to Social Security and Medicare tax under IRC §3121(v)(1)(A), so the base is the full $100,000 and the tax is $7,650." +us,scenario_110,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $6,889 employer-sponsored insurance premium as a pre-tax deduction that lowers FICA wages to $93,111. Employee FICA applies to the full $100,000 of stated gross wages, giving $7,650." +us,scenario_110,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It correctly kept the 401(k) deferral in FICA wages but wrongly took out $6,889 of employer-plan premiums, which gave $93,111. The payroll-tax base is the full $100,000, so the employee tax is $6,200 + $1,450 = $7,650." +us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It excluded $6,889 of employer-sponsored health premiums from FICA wages as a pre-tax deduction, but the facts list no salary reduction. Social Security at 6.2% and Medicare at 1.45% on the full $100,000 give $7,650." +us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no answer to score. The correct figure is 7.65% of the $100,000 gross wages, or $7,650." +us,scenario_110,payroll_tax,minimax-m3,llm_error,other,False,"It claimed PolicyEngine returned 0 and submitted zero payroll tax, ignoring $100,000 of wage income that is fully subject to employee FICA. The 6.2% Social Security and 1.45% Medicare rates yield $7,650." +us,scenario_110,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"It reduced payroll-tax wages by the $6,889 employer-sponsored insurance premium to $93,111. The stated gross wages of $100,000 are the FICA base, so the employee tax is $7,650." +us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly computed $6,200 Social Security plus $1,450 Medicare = $7,650, but it submitted the unrelated figure $9,351.90. The final value contradicts its own derivation." +us,scenario_110,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It added a $252.55 Additional Medicare Tax even though the only wages are $100,000, well below the $200,000 single-filer threshold. The 0.9% tax applies only to wages and self-employment income, not investment income, so the correct total is $6,200 + $1,450 = $7,650." us,scenario_110,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the 2024 base of $360.69 instead of the 2026 base of $332, a ~$1,700 exemption instead of $1,900, and an AGI that deducted the phased-out $198 IRA, which gave $4,265.55. It then submitted $5,052, a number its own derivation does not support." -us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its taxable income of $168,043 is correct, but it applied 2.75% only to the amount above $26,050 and left out the $332 base in the 2026 Ohio schedule. That gives $3,905 instead of $332 + $3,904.81 = $4,236.81." -us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted a federal-style $15,000 standard deduction, which Ohio does not have, instead of the $1,900 personal exemption. It also deducted the phased-out IRA contribution and guessed at progressive brackets instead of applying $332 + 2.75% above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It submitted $0 without doing any calculation, even though $168,042.94 of Ohio taxable income produces $332 + 2.75% × $141,992.94 = $4,236.81." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used invented indexed brackets with a 3.5% rate above $103,500, which no longer exists in 2026, and reached ~$4,328. It then dropped to an arbitrary $2,900 with no derivation instead of applying $332 + 2.75% over $26,050 to $168,042.94." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied 3.5% to all income above $26,050 on top of the 2024 base of $360.69, which gave $5,325. It then submitted an unexplained $4,517. The 2026 schedule is a flat 2.75% plus a $332 base, applied to $168,042.94 after the $1,900 exemption." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted the $954 tax-exempt interest even though it was never in AGI, and used a ~$2,500 exemption instead of $1,900. Its $3,900 matches a flat 2.75% above $26,050 with the $332 base left out." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 schedule, taxing only 2.75% of the excess over $26,050. It also deducted the $198 traditional IRA contribution, which is fully phased out given workplace-plan coverage and AGI near $170k, so its taxable income came out $198 too low." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted $10,915 of itemized deductions, which Ohio does not allow, instead of the $1,900 personal exemption. It applied 3.5% instead of the 2026 rate of 2.75% plus the $332 base, and took the $20 exemption credit, which is unavailable at this income." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used invented graduated brackets (3.5% and ~3.99%) instead of the 2026 schedule of $332 + 2.75% above $26,050. It also subtracted nonrefundable credits (retirement income and others) that this household does not qualify for, ending at $3,200." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It wrongly skipped the personal exemption, which is $1,900 for income above $80k rather than zero. It also left out the $332 base and taxed only 2.75% of AGI above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used old graduated rates (2.765%/3.302%/3.688%) that have been repealed, deducted the phased-out IRA, and wrongly treated the $1,900 exemption as phased out. The 2026 schedule is $332 + 2.75% above $26,050 on $168,042.94." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied 2.75% to the entire taxable income with no $26,050 zero bracket and no $332 base, and left out the $1,900 exemption. Its AGI of $163,054 wrongly subtracts the $6,889 employer insurance premiums, and it added back $954 of tax-exempt interest, which the Ohio AGI calculation does not include." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $162,856 subtracts both the $6,889 employer insurance premiums and the phased-out $198 IRA from the correct $169,942.94. It then applied the repealed 2.75%/3.5% two-bracket schedule without the $332 base." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $2,400 exemption instead of the $1,900 that applies above $80k, and invented a three-component graduated calculation instead of $332 + 2.75% × ($168,042.94 − $26,050)." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the 3.5% rate above $100,000, which was eliminated for 2026, without the $332 base. It also used a $2,500 exemption instead of $1,900." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It says it subtracted 'standard deductions', but Ohio has none; Ohio allows only the $1,900 personal exemption. Its $2,981 is $1,256 short, which matches subtracting about $34k of nonexistent deductions and leaving out the $332 base." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $163,054, which wrongly subtracts the $6,889 employer insurance premiums from wages; the correct AGI is $169,942.94. It applied progressive brackets instead of the 2026 schedule of $332 + 2.75%, and its $4,241 lands near the reference only because these errors offset." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a progressive schedule with a 3.5% top rate, which has been repealed, instead of the 2026 schedule of $332 + 2.75% above $26,050. That produced $3,850 instead of $4,236.81." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. Its $3,280 is $957 short of $332 + 2.75% × ($168,042.94 − $26,050), which matches leaving out the $332 base and subtracting deductions beyond the $1,900 exemption." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,240.52 is $3.71 above the correct $332 + 2.75% × $141,992.94, which means it taxed about $135 more than the correct $168,042.94 of taxable income." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no derivation. Its $3,986 matches 2.75% above $26,050 on about $171k with no $1,900 exemption and no $332 base, instead of $332 + 2.75% × ($168,042.94 − $26,050)." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It wrongly claimed there is no personal exemption above $80,000; the exemption is $1,900 at that level. It also left out the $332 base, taxing only 2.75% of AGI above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the repealed 3.5% rate above $100,000 and left out the $332 base. It also wrongly treated the personal exemption as fully phased out, when $1,900 applies above $80k." -us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2024-style schedule with 3.5% above $100,000, which is repealed for 2026, and left out the $332 base. It also skipped the $1,900 personal exemption." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It admitted the tax is positive but submitted $0 anyway, instead of computing $332 + 2.75% × ($168,042.94 − $26,050) = $4,236.81." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted itemized deductions (mortgage interest, charity), which Ohio does not allow; Ohio allows only the $1,900 personal exemption. Its $5,625 overshoots by $1,388, which matches graduated rates above the 2026 flat 2.75% plus $332 base." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly found $168,043 of taxable income, but taxed only 2.75% of the excess over $26,050 and left out the $332 base in the 2026 schedule." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $4,418 matches the repealed 2.75%/3.5% two-bracket schedule without the $332 base, applied to roughly $168k of taxable income. The 2026 rate is a flat 2.75% plus $332 above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It applied 2.75% only to income above the zero bracket and left out the $332 base in the 2026 Ohio schedule, which puts it $332 low." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 schedule and used a $1,850 exemption instead of $1,900." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied the $332 + 2.75% schedule correctly, but its AGI of $163,054 wrongly subtracts the $6,889 employer insurance premiums. It also added back $954 of tax-exempt interest, which the Ohio AGI calculation does not include, leaving taxable income at $162,108 instead of $168,042.94." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It used the repealed 3.5% bracket above $100,000 without the $332 base. It also skipped the $1,900 personal exemption." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It skipped the $1,900 personal exemption and left out the $332 base, taxing only 2.75% of AGI above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its rate schedule ($332 + 2.75% above $26,050) and $1,900 exemption are correct. But its AGI of $163,054 wrongly subtracts the $6,889 employer insurance premiums from wages, which understates taxable income by $6,889." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no real derivation and subtracted 'modest credits' this household does not qualify for. Its $3,200 is $1,037 short of $332 + 2.75% × ($168,042.94 − $26,050)." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its $168,043 of taxable income is correct, but it applied the 2024-style 3.5% bracket above $100,000, which is repealed for 2026, and left out the $332 base." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly found $168,043 of taxable income, but computed only 2.75% × $141,993 and left out the $332 base in the 2026 schedule." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It added back $954 of tax-exempt interest, which the Ohio AGI calculation does not include. It also applied the repealed 3.5% bracket above $100,000 without the $332 base." -us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It left out the $332 base. It also used a $2,400 exemption instead of the $1,900 that applies above $80k, and deducted the phased-out $198 IRA from AGI." -us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used the 2024 schedule ($2,394 at $100,000 plus 3.5% above), but for 2026 the 3.5% bracket is gone and all income above $26,050 is taxed at 2.75% plus a $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Its own working reached $4,265.55, but it submitted $5,052 with no derivation behind that number. The working itself was also wrong: it used the old $360.69 base instead of the 2026 $332 base, deducted the $198 IRA contribution even though that deduction is phased out, and used a $1,700 exemption instead of $1,900." +us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got AGI ($169,943), the $1,900 exemption and taxable income ($168,043) right. It then applied 2.75% only to the amount over $26,050 and left out the $332 base, so it reached $3,905 instead of $332 + $3,904.81 = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted a $15,000 federal-style standard deduction, but Ohio has no standard deduction and allows only the $1,900 personal exemption. It also deducted the phased-out $198 IRA contribution and guessed a progressive bracket result instead of applying $332 + 2.75% of taxable income over $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It did no calculation and entered $0, even though Ohio taxable income of $168,043 produces $332 + 2.75% × $141,993 = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an indexed $27,350 zero bracket and a 3.5% rate above $103,500, but neither applies in 2026; the 2026 rate is a flat 2.75% above $26,050 plus a $332 base. It then dropped its own $4,328 result and entered an arbitrary $2,900." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied 3.5% to all income over $26,050 on top of the old $360.69 base, getting $5,325, then submitted an unexplained $4,517. The 2026 schedule is $332 + 2.75% of the excess, and it also deducted the phased-out $198 IRA contribution." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted the $954 of tax-exempt interest even though that interest was never in AGI, and used a $2,500 exemption instead of $1,900. Its $3,900 result matches 2.75% of the excess over $26,050 with the $332 base left out." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It deducted the $198 IRA contribution even though that deduction is phased out for a high-income active plan participant, giving AGI of $169,745 instead of $169,943. It also applied 2.75% without the $332 base amount in Ohio's 2026 schedule." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted $10,915 of itemized deductions, but Ohio has no itemized deductions; only the $1,900 personal exemption reduces taxable income. It also applied a 3.5% rate instead of 2.75%, left out the $332 base, and took a $20 exemption credit that only applies when Ohio taxable income is under $30,000." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It invented higher brackets (3.5% and roughly 3.99%) and nonrefundable credits that the household does not qualify for. It also added back the $954 of tax-exempt interest, which does not change Ohio AGI here. Its $3,200 does not follow from the correct $332 + 2.75% × ($168,043 − $26,050) = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It wrongly assumed there is no personal exemption at this income, but the $1,900 exemption applies above $80,000 of Ohio AGI. It also left out the $332 base and taxed $143,893 at 2.75% instead of computing $332 + 2.75% × $141,993." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used outdated graduated rates (2.765%, 3.302% and 3.688%) instead of the 2026 schedule of $332 + 2.75% above $26,050. It also wrongly treated the exemption as fully phased out instead of subtracting $1,900, and deducted the phased-out IRA contribution." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took $6,889 of employer insurance premiums out of AGI (giving $163,054) and added back $954 of tax-exempt interest. It then applied 2.75% to all income, ignoring both the $26,050 zero bracket and the $1,900 exemption." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $162,856 subtracts both the $6,889 of employer insurance premiums and the phased-out $198 IRA deduction, and it then added back $954 of tax-exempt interest. It also used a 3.5% bracket above $100,000 that no longer applies and left out the $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the $2,400 exemption for incomes up to $40,000 instead of the $1,900 that applies above $80,000. It also used a made-up three-tier schedule with higher marginal rates instead of $332 + 2.75% of taxable income over $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a $2,500 exemption instead of $1,900 and taxed income above $100,000 at 3.5%, a bracket that no longer applies in 2026. It also left out the $332 base, so it did not compute $332 + 2.75% × ($168,043 − $26,050)." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It applied unspecified standard deductions, which Ohio does not have, and gave no derivation. Its $2,981 is well below the correct $332 + 2.75% × ($169,943 − $1,900 − $26,050) = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from AGI of $163,054, which wrongly subtracts $6,889 of employer insurance premiums from the correct $169,943. It then applied progressive brackets instead of the 2026 schedule of $332 + 2.75% above $26,050, so its result being near the reference is a coincidence." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied progressive brackets with a 3.5% top rate, but 2026 Ohio tax is $332 + 2.75% of taxable income over $26,050. Its $3,850 also does not reflect taxable income of $168,043 after the $1,900 exemption." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave $3,280 with no derivation. The correct calculation is taxable income of $168,043 taxed at $332 + 2.75% × $141,993 = $4,236.81, so its number understates both the base amount and the taxable income." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $4,240.52 is $3.71 above the correct $332 + 2.75% × $141,992.94 = $4,236.81. That gap equals about $135 of extra Ohio taxable income, which fits an exemption smaller than the $1,900 that applies above $80,000 of Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave $3,986 with no derivation, which fits a plain 2.75% rate on the excess over $26,050 without the $332 base in Ohio's 2026 schedule. The correct tax on $168,043 of taxable income is $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It wrongly claimed there is no personal exemption above $80,000, but a $1,900 exemption applies at that level. It also left out the $332 base, taxing ($169,943 − $26,050) at 2.75% instead of computing $332 + 2.75% × $141,993." +us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It got AGI ($169,943) and the IRA phase-out right but wrongly treated the exemption as fully phased out instead of subtracting $1,900. It also taxed income above $100,000 at 3.5%, a bracket that no longer applies in 2026, and left out the $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It never subtracted the $1,900 personal exemption and used a 3.5% bracket above $100,000 that no longer applies in 2026. Under the correct schedule, $332 + 2.75% of the $141,993 above $26,050 gives $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"It acknowledged the tax was positive but declined to calculate it and entered $0. The full calculation is simple: $168,043 of taxable income gives $332 + 2.75% × $141,993 = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted itemized amounts such as mortgage interest and charitable gifts, but Ohio does not allow itemized deductions. Its $5,625 implies rates far above Ohio's 2026 schedule of $332 + 2.75% over $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got AGI ($169,943), the $1,900 exemption and taxable income ($168,043) right. It then taxed only the excess over $26,050 at 2.75% and left out the $332 base, so it reached $3,905 instead of $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $4,418 matches 2.75% from $26,050 to $100,000 plus 3.5% above $100,000 on about $168,000 of taxable income. That 3.5% bracket no longer applies in 2026, and the $332 base was left out." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It applied 2.75% only to taxable income above the $26,050 zero bracket and left out the $332 base in Ohio's schedule, so it reached $3,905 instead of $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a $1,850 exemption instead of $1,900 and, more importantly, left out the $332 base. The correct tax is $332 + 2.75% × ($168,043 − $26,050) = $4,236.81, not 2.75% of the excess alone." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used the correct $332 + 2.75% schedule but started from AGI of $163,054, which wrongly subtracts $6,889 of employer insurance premiums. It also added back $954 of tax-exempt interest, so taxable income came out at $162,108 instead of $168,043." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It taxed the full $169,943 without subtracting the $1,900 personal exemption. It also used a 3.5% bracket above $100,000 that no longer applies in 2026 and left out the $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It skipped the $1,900 personal exemption and applied 2.75% to the excess over $26,050 without the $332 base. The correct figure is $332 + 2.75% × $141,993 = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used the correct $332 + 2.75% schedule and the $1,900 exemption, but its AGI of $163,054 wrongly subtracts $6,889 of employer insurance premiums. Correct AGI is $169,943, giving taxable income of $168,043." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave $3,200 with no derivation, citing credits that this household does not qualify for at this income. The correct tax on $168,043 of taxable income is $332 + 2.75% × $141,993 = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It got taxable income ($168,043) right but applied 2024-style brackets with 3.5% above $100,000 and no $332 base. In 2026 the top bracket is gone, and the schedule is $332 + 2.75% of taxable income over $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got AGI, the $1,900 exemption and taxable income of $168,043 right. It then applied 2.75% to the excess over $26,050 and left out the $332 base, so it reached $3,905 instead of $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It added back $954 of tax-exempt interest, which does not change Ohio AGI here, and taxed income above $100,000 at 3.5%, a bracket that no longer applies in 2026. It also left out the $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It deducted the phased-out $198 IRA contribution and used the $2,400 low-income exemption instead of $1,900. It also left out the $332 base, so taxable income and the tax both came out low." +us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used the old schedule, with $2,394 on the first $100,000 (the $360.69 base plus 2.75%) and 3.5% above that. The 2026 schedule is $332 + 2.75% on all taxable income above $26,050, and its exemption was also too large." us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for state_income_tax_before_refundable_credits. -us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It left out the $332 base and subtracted the $20 personal exemption credit, which is limited to low-income filers. It also deducted the phased-out $198 IRA from AGI." -us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It falsely claimed PolicyEngine returned $0. The $168,042.94 of Ohio taxable income produces $332 + 2.75% × $141,992.94 = $4,236.81." -us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 schedule and used a $1,700 exemption instead of the $1,900 that applies above $80k." -us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It added $954 of tax-exempt interest into AGI and used invented 2.765%/3.225% brackets. It then subtracted an invented $250 exemption credit and further invented 'rate reductions', instead of $332 + 2.75% above $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It cut taxable income to $96,517 by subtracting about $71,500 of standard deductions that Ohio does not have. The correct base is $169,942.94 AGI minus only the $1,900 exemption, which is $168,042.94." +us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It deducted the phased-out $198 IRA contribution, left out the $332 base, and subtracted a $20 exemption credit. That credit is only available when Ohio taxable income is under $30,000." +us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It wrongly claimed the engine returned $0 and did no calculation. Ohio taxable income of $168,043 produces $332 + 2.75% × $141,993 = $4,236.81." +us,scenario_110,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its taxable income of $161,154 comes from AGI of $163,054, which wrongly subtracts $6,889 of employer insurance premiums. It also left out the $332 base, applying only 2.75% to the excess over $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,700 exemption instead of the $1,900 that applies above $80,000 of Ohio AGI. It also left out the $332 base, applying only 2.75% to the excess over $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It wrongly included $954 of tax-exempt interest in AGI, skipped the $1,900 exemption, and used old 2.765%/3.225% brackets. It then took a $250 exemption credit that does not exist and made an arbitrary further cut down to $3,148.93." +us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reported Ohio taxable income of $96,517, which leaves out most of the $74,188 of interest, dividends, capital gains and estate income included in Ohio AGI. Correct taxable income is $168,043, giving $4,236.81." us,scenario_110,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated about $16,600 of Social Security as taxable even though its own stated combined income of $30,940 is below the $32,000 joint-filer threshold, not above the $44,000 upper threshold. It then calculated zero taxable income after the standard deduction but replaced the resulting $0 tax with an unsupported $1,362 estimate." us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $5,405.50 of Social Security in AGI, but still correctly found that the standard deduction reduced taxable income and tax liability to $0. It then submitted $1,400 despite deriving no taxable income and identifying no tax or credit computation that produces that amount." @@ -9012,100 +9127,102 @@ us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base, us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly stated that passive real-estate farm rent is not subject to self-employment tax, then contradicted that rule by calculating tax as though the entire $1,920 were farm-operator self-employment earnings. The actual net self-employment tax base is zero because the household has no positive net self-employment earnings." us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model improperly combined farm rent, financial assistance, and the capital loss into a self-employment tax base. Financial assistance and capital gains or losses are not self-employment earnings, and the household’s relevant net earnings are nonpositive, so no self-employment tax applies." us,scenario_112,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the full $1,920 of farm rent as net self-employment income and applied the standard 92.35% and 15.3% factors without establishing positive net earnings from self-employment. The resulting tax base is zero, and its submitted $269 also does not match its own stated calculation of about $271." -us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added the $1,920 farm rent to countable income, which raised net income to about $1,429 and failed the 100% FPL net test. It also first applied a 130% gross test even though Texas BBCE covers the household. PolicyEngine's SNAP income is wages plus financial assistance: net is $1,269.19 (97% FPL), and the one-person minimum allotment of about $24/month ($287.68/yr) applies, not $0." -us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Counted the $160/month of farm rent in SNAP income, so net income came out near $1,429 and failed the net test. It then let the formula go to zero because 30% of net exceeded $298. PolicyEngine excludes farm rent, so net is $1,269.19 (97% FPL), and an eligible one-person household gets the minimum allotment of about $24/month ($287.68/yr) instead of $0." -us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Applied the standard 130% FPL gross income limit and missed that Texas TANF-funded BBCE confers categorical eligibility with a 165% FPL gross limit. The household's gross income of $1,576.90 (121% FPL) is within that limit, net income passes at 97% FPL, and the household gets the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Declared the household not established as eligible without running any income test. The household passes Texas's BBCE gross test (121% FPL) and the net test ($1,269.19, 97% FPL), so it is eligible. Because 30% of net exceeds the $298 maximum, it receives the one-person minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Disqualified the household with a 130% FPL gross income test and ignored Texas BBCE through TANF non-cash assistance, which raises the gross limit to 165% FPL. Gross income passes at 121% FPL and net passes at 97% FPL, so the household gets the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Treated the $13,000 financial assistance as an excluded gift and counted only wages and farm rent, which made net income low and produced a large benefit ($2,304). PolicyEngine counts the financial assistance as unearned income, so net income is $1,269.19. Thirty percent of that ($380.76) exceeds the $298 maximum, leaving only the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Left out the $13,000 financial assistance from countable income and assumed a near-maximum benefit of about $275/month. With the assistance counted as unearned income, net income is $1,269.19 and the expected contribution of $380.76 exceeds the $298 maximum. The household therefore receives only the minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly invoked Texas BBCE but assumed the $13,000 financial assistance does not count, which gave $346 of net income and a $194/month benefit. PolicyEngine counts the assistance ($1,083.33/mo) as unearned income, so net income is $1,269.19. The 30% contribution then exceeds the $298 maximum, and only the one-person minimum allotment ($287.68/yr) remains." -us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Applied a 130% FPL gross income limit of about $1,632/month and concluded the household was ineligible, missing Texas TANF-funded BBCE (165% FPL gross limit). Gross income of $1,576.90 (121% FPL) passes, net income of $1,269.19 passes the 100% FPL test, and the one-person minimum allotment applies ($287.68/yr)." -us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Used the 130% FPL gross income threshold to deny eligibility and never applied Texas BBCE, which confers categorical eligibility up to 165% FPL. The household passes at 121% FPL gross and 97% FPL net, and receives the one-person minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $13,000 financial assistance from countable income and computed $194/month. PolicyEngine counts it as unearned income, so net income is $1,269.19 and 30% of net ($380.76) exceeds the $298 maximum. The household gets only the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Failed the household on the 130% FPL gross income test and ignored Texas BBCE, which waives that test up to 165% FPL through TANF non-cash assistance. The household is categorically eligible, passes the net test at 97% FPL, and gets the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the 130% FPL gross limit (about $1,672/month) and stopped there, missing Texas BBCE's 165% FPL gross limit. SNAP gross income of $1,576.90 (121% FPL) passes and net income of $1,269.19 passes, so the minimum one-person allotment of about $24/month ($287.68/yr) is paid." -us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum allotment but then set the benefit to $0. Eligible one- and two-person households receive the minimum allotment (8% of the $298 maximum, about $24/month), which gives $287.68 for the year." -us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Computed a garbled net income ($6,454 with a $1,936 contribution) and zeroed the benefit because the contribution exceeded the maximum allotment. The household's monthly net income is $1,269.19, which passes the net test. Because it is an eligible one-person household, it gets the minimum allotment of about $24/month ($287.68/yr), not $0." -us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Included the $1,920 farm rent in SNAP income, which put net income at $1,440.20, and then failed the household on the net income test. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL) and passes. The categorically eligible one-person household gets the minimum allotment ($287.68/yr)." -us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Asserted ineligibility without any computation. The household is categorically eligible through Texas BBCE (gross 121% FPL), passes the net test at 97% FPL, and receives the one-person minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit because 30% of net income exceeds the one-person maximum and never applied the SNAP minimum benefit. An eligible one-person household whose formula benefit falls below the floor receives the minimum allotment of about $24/month, which totals $287.68 for the year." -us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, which put net income at $1,440, and reduced the benefit to $0 because 30% of net exceeded the maximum allotment. That skips the minimum allotment for eligible one-person households. PolicyEngine's net income is $1,269.19 (eligible), and the floor of about $24/month yields $287.68/yr." -us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Reported $0 with no derivation, which matches letting the benefit formula go to zero. The household is eligible: gross 121% FPL under Texas BBCE and net 97% FPL. Because 30% of net exceeds the $298 maximum, the one-person minimum allotment of about $24/month ($287.68/yr) applies." -us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Concluded $0 because 30% of net income exceeds the one-person maximum allotment and omitted the minimum benefit rule. Eligible one-person households receive at least 8% of the maximum allotment, about $24/month, for $287.68 a year." -us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Claimed net income exceeds the limit, which only holds if farm rent is counted. PolicyEngine's SNAP income is wages plus financial assistance, giving net income of $1,269.19 (97% FPL). The household passes and receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit because 30% of net income exceeds the maximum allotment, and also cited the vehicle and assets even though the asset test is met ($10 bank account). The eligible one-person household receives the minimum allotment of about $24/month, $287.68 for the year." -us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Counted the $1,920 farm rent, which put net income at about $1,434, above 100% FPL, and declared the household ineligible. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL) and passes. The one-person minimum allotment then yields $287.68/yr." -us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Used a 130% FPL gross limit understated at about $17,000 and also treated the $11,838 vehicle as breaking the asset limit. Texas BBCE raises the gross limit to 165% FPL, and the household passes the asset test with $10 in the bank. The household is therefore eligible and receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Asserted that income and resources are too high with no supporting computation. The household passes the Texas BBCE gross test (121% FPL), the net test (97% FPL), and the asset test ($10), so it is eligible. It receives the one-person minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 on the claim that the facts do not support a benefit computation, even though wages, financial assistance, and household size fully determine SNAP. The single-person household is eligible at 97% FPL net income and receives the minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Applied the 130% FPL gross income limit and ignored Texas TANF-funded BBCE, which sets a 165% FPL gross limit. The household is categorically eligible, passes the net test at 97% FPL, and gets the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Failed the household on the standard gross-income limit without applying Texas BBCE, under which gross income of 121% FPL passes the 165% FPL limit. With net income at 97% FPL, the household is eligible and receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and concluded the household fails the net income test. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL), below the $1,304.17 limit. The household is eligible, and its benefit is the one-person minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Applied the ordinary one-person gross income limit and missed Texas BBCE's 165% FPL threshold through TANF non-cash assistance. Gross income passes at 121% FPL and net income passes at 97% FPL, so the minimum one-person allotment ($287.68/yr) is paid." -us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Included farm rent in countable income and concluded that net income exceeds the 100% FPL limit. PolicyEngine counts only wages and financial assistance, so net income is $1,269.19 (97% FPL). The household is eligible and receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent alongside wages and financial assistance, which pushed net income above the limit. PolicyEngine excludes farm rent, so net income is $1,269.19 (97% FPL) and passes. Thirty percent of net exceeds the $298 maximum, so the one-person minimum allotment of about $24/month ($287.68/yr) applies." -us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"Concluded that income is too high to produce a positive benefit and omitted the minimum allotment rule. The household is eligible at 97% FPL net income, and although the formula benefit is negative, eligible one-person households get about $24/month ($287.68/yr)." -us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Treated the formula's non-positive result as no benefit and never applied the SNAP minimum allotment for one-person households. The household is eligible (net $1,269.19, 97% FPL), so it receives about $24/month, $287.68 for the year." -us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Asserted that income and assets exceed SNAP limits without computing either. Assets are $10, gross income is 121% FPL (within Texas BBCE's 165% limit), and net income is 97% FPL, so the household is eligible for the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Declared ineligibility on the gross or net test without applying Texas BBCE, under which gross income of 121% FPL passes, while net income of $1,269.19 is 97% FPL. The eligible one-person household gets the minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the one-person maximum allotment but set SNAP to $0 instead of applying the minimum allotment. Eligible one-person households receive 8% of the maximum, about $24/month, totaling $287.68." -us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Counted the $1,920 farm rent in SNAP income, so net income landed above 100% FPL and the household was found ineligible. PolicyEngine's SNAP income excludes farm rent, giving net income of $1,269.19 (97% FPL). The household is eligible and receives the minimum one-person allotment ($287.68/yr)." -us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Compared gross income against a 130% FPL limit (about $20,700) and missed Texas BBCE, whose 165% FPL gross limit the household easily meets. Net income passes at 97% FPL, so the household receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Applied the 130% FPL gross income limit and ignored Texas TANF non-cash BBCE, which waives that test up to 165% FPL. The household is categorically eligible, passes the net test at 97% FPL, and receives the minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"Required gross income at or below 130% FPL and never applied Texas BBCE, which confers categorical eligibility with a 165% FPL gross limit. Gross income is 121% FPL and net income is 97% FPL, so the one-person minimum allotment ($287.68/yr) applies." -us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Stated that the household lacks TANF/SSI categorical eligibility, but Texas BBCE confers categorical eligibility through TANF non-cash assistance. It also let the expected contribution eliminate the allotment. An eligible one-person household gets the minimum allotment of about $24/month ($287.68/yr) even when 30% of net income exceeds the $298 maximum." -us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed that a single wage-earning adult does not qualify for SNAP, but one-person households are eligible under the standard income tests. This household passes Texas BBCE (gross 121% FPL) and the net test (97% FPL), and receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Failed the household on a 130% FPL gross limit of about $1,695/month and missed Texas BBCE's 165% FPL gross limit. The household is categorically eligible, passes the net test at $1,269.19 (97% FPL), and receives the one-person minimum allotment ($287.68/yr)." -us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the maximum allotment, then dropped that result and submitted $3,576, the full $298/month maximum, by invoking an unlisted shelter deduction. With no shelter costs, the benefit formula is negative, and the eligible one-person household receives only the minimum allotment of about $24/month ($287.68/yr)." -us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Declared the household ineligible with no computation. It is categorically eligible through Texas BBCE (gross 121% FPL), passes the net test (97% FPL), and receives the one-person minimum allotment of about $24/month ($287.68/yr)." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $19,518. That total left Virginia income tax out of SALT, counted the $2,000 of OTC expenses but not the Medicare Part B premium in medical, and skipped the 0.5% charitable floor. Those choices gave taxable income of $78,077 and tax of about $11,889. It then cut that to $8,971 with an unexplained 'estimation' step that its own computation does not support." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"The structure was correct (itemize, senior deduction $4,106, QBI $4,872, 2026 brackets), but itemized deductions came to only $22,510. It estimated Virginia income tax at $3,524 instead of about $5,367, and it counted OTC expenses but not the $2,434.80 Medicare Part B premium in medical. That left taxable income $2,277 too high, at $75,085." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income and used an invented $28,300 standard deduction. It also refused to itemize, which contradicts its own comparison. It then added $336 of Medicare payroll tax to income tax and an arbitrary $2,000 'safe harbor' adjustment, so $10,268 does not come from any valid tax computation." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed taxable income falls in a zero-tax range. This ignored about $106,573 of AGI ($86,744 of non-Social Security income plus 85% of benefits) that leaves $72,807 of taxable income after all deductions." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It entered SALT as the full $10,000 cap when only $4,064 of real estate tax was listed. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 MAGI and left out the Medicare Part B premium from medical. After reaching $10,711, it submitted $5,489 with no supporting derivation." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $4,500 of mortgage interest even though no interest was listed. It left Virginia income tax out of SALT and the Part B premium out of medical, and it omitted the OBBBA senior deduction ($4,105.65) entirely. It computed $11,802, then lowered it to $11,210 with an unexplained adjustment." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a nonexistent half-SE-tax deduction of $1,721 for the partnership income. It took the standard deduction plus an unphased $6,000 senior bonus instead of itemizing $24,787. Its own math reached about $11,900, and it then submitted $6,900 with no supporting computation." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The method was correct, but SALT was understated at $7,618.50 (Virginia income tax estimated at about $3,554 instead of about $5,367). Medical was $4,007 because it counted OTC expenses and not the $2,434.80 Part B premium. Itemized deductions were about $2,248 short, which left taxable income at $75,055 instead of $72,807." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $6,693 of mortgage interest from the loan balance although no interest was listed. It left out Virginia income tax and the Part B premium, skipped the charitable floor, and omitted the OBBBA senior deduction. It also used self-inflated 2025 brackets instead of the 2026 schedule." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 phased senior deduction. It also itemized only $19,518, with no Virginia income tax in SALT and OTC counted instead of the Part B premium in medical. That produced taxable income of $82,183 instead of $72,807." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only about $21,188 instead of $24,787. It underestimated Virginia income tax in SALT and left the $2,434.80 Medicare Part B premium out of medical. Taxable income ended about $3,600 too high, at $76,407." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted $1,721 of half-SE-tax that does not apply to this partnership/S-corp income. It omitted the senior deduction and left Virginia income tax and the Part B premium out of itemized deductions. Taxable income came to $82,357." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65) and counted only $10,000 of medical, missing the $2,434.80 Part B premium. It also understated Virginia income tax at $3,551.53. Taxable income came to $80,631 instead of $72,807." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired pre-TCJA regime for 2026: a $5,350 personal exemption and 10/15/25% brackets, with no QBI deduction and no senior deduction. OBBBA made the TCJA rate structure permanent and added the senior deduction." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $79,373 is about $6,565 above the correct $72,807. That gap matches omitting the $4,105.65 phased senior deduction and understating itemized deductions (Virginia income tax and the Medicare Part B premium in medical)." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used a pre-TCJA $5,000 personal exemption and 10/15/25% brackets instead of the permanent OBBBA 2026 brackets. It left out the QBI and senior deductions and also subtracted a nonexistent $1,721 half-SE-tax adjustment." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated gross income at $110,094 instead of AGI of $106,573 and omitted the senior deduction. Taxable income came to about $81,300. Its $13,919 is also more than the 2026 10/12/22% brackets produce on that amount, so its rates were wrong as well." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly applied 2026 pre-TCJA law with personal exemptions and sunset brackets. That leaves out the QBI deduction, the OBBBA senior deduction, and the permanent 10/12/22% rate schedule." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used 'sunset' brackets and a $5,050 personal exemption, which no longer exist after OBBBA. It omitted the $4,105.65 senior deduction and itemized only $19,256, without the Virginia income tax and the Part B premium." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The correct path yields taxable income of $72,807.33 and tax of $10,729.61. Its $9,334 corresponds to taxable income of about $66,460 under the 2026 brackets, so it overstated deductions by roughly $6,300." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65). It itemized only $22,018 because it underestimated Virginia income tax and left out the Medicare Part B premium, which left taxable income at $79,682." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction even though itemized deductions of $24,787 (SALT with Virginia income tax, medical with the Part B premium, charity) are larger. It also left out the phased senior deduction, so taxable income came to about $83,550." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $4,872 QBI deduction or the $4,105.65 senior deduction. It itemized only $17,518, with no Virginia income tax in SALT and no Part B premium in medical, which put taxable income near $89,055." -us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It asserted that the QBI deduction lapsed after 2025 and applied a personal exemption with 10/15/25% brackets. OBBBA made both the QBI deduction and the TCJA rate schedule permanent and added the senior deduction, which it also omitted." -us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction. It itemized only $17,518, with no Virginia income tax in SALT and no Part B premium or OTC in medical. Taxable income came to $84,183 instead of $72,807." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard and age-65 deductions offset all income. That ignores about $106,573 of AGI, which leaves $72,807 of taxable income even after itemized, QBI and senior deductions." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the income was mainly Social Security with modest taxable income. It ignored $86,744 of wages, partnership, pension and IRA income, which also makes 85% of benefits taxable." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'post-2025' pre-TCJA brackets and a personal exemption instead of the permanent OBBBA 10/12/22% schedule. It also omitted the QBI and senior deductions and itemized only $17,518." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of an imputed self-employment tax that does not apply to this partnership/S-corp income. It also took the standard-deduction path, and its overstated deductions gave taxable income of $67,020 instead of $72,807." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction ($19,150 total). Itemized deductions of $24,787 are larger and should have been used. That choice left taxable income at $78,445 instead of $72,807." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $11,540 implies taxable income of about $76,490, roughly $3,680 above the correct amount. That matches understated itemized deductions from leaving out the full Virginia income tax in SALT and the Medicare Part B premium in medical." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"The senior deduction ($4,105.63), QBI deduction and brackets were all correct, but itemized deductions were only $20,639 instead of $24,787. It underestimated Virginia income tax in SALT and left the $2,434.80 Part B premium out of medical." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $80,675 is about $7,870 too high. It omitted the $4,105.65 senior deduction and understated itemized deductions by leaving out the full Virginia income tax and the Medicare Part B premium." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped $10,000 from AGI ($96,572.80 instead of $106,572.80), which inflated the senior deduction to $4,705.63. It also took the standard plus $1,000 non-itemizer charitable deduction instead of itemizing $24,787." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The senior deduction, QBI deduction and 2026 brackets were correct, but itemized deductions were about $4,148 too low. It underestimated imputed Virginia income tax in SALT and excluded the $2,434.80 Medicare Part B premium from medical." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The correct taxable income is $72,807.33. Its $10,234 corresponds to taxable income of about $70,555 under the 2026 brackets, so it overstated deductions by roughly $2,250." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law: a $5,468 personal exemption, restored 10/15/25% brackets, and no QBI or senior deduction. OBBBA kept the 10/12/22% schedule and the QBI deduction permanently and added the senior deduction." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset and declared the QBI deduction expired. It used a $5,300 personal exemption and 10/15/25% brackets and left out the OBBBA senior deduction, none of which matches 2026 law." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded the $18,150 standard deduction beats itemizing. Itemized deductions are $24,787 once SALT includes Virginia income tax and medical includes the Part B premium, so taxable income should have been $72,807 instead of $79,445." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65). It understated itemized deductions by estimating Virginia income tax at $2,746 and leaving out the Part B premium, which left taxable income at $79,437." -us,scenario_114,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets and a $5,300 personal exemption instead of the permanent OBBBA schedule. It omitted the QBI and senior deductions and subtracted a nonexistent half-SE-tax adjustment from AGI." -us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score." -us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It computed itemized deductions as only $17,518, leaving Virginia income tax out of SALT and the Part B premium out of medical. As a result it took the standard plus non-itemizer charitable deduction instead of itemizing $24,787, and taxable income came to $78,445." -us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It acknowledged positive taxable income but still reported $0 tax with no credit to justify it. It also invented mortgage interest, while about $72,807 of taxable income at 2026 brackets produces $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated itemized deductions at about $19,500, leaving Virginia income tax out of SALT and the Medicare Part B premium out of medical. It therefore chose the standard deduction, which put taxable income at $79,445 instead of itemizing $24,787." -us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left all $19,829 of taxable Social Security out of income and used an invented $15,750 standard deduction. It never considered itemizing, and its reported $15,654 is far above what the 2026 brackets produce even on its own $66,122 taxable income." -us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It reported AGI as $80,164, which does not match its own listed components that sum to $106,573. It cut the QBI deduction to $3,550 and took a $20,400 standard deduction instead of itemizing. It then subtracted invented reductions that treat deductions as credits, going from $7,761 to $5,290." +us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"The model added the $1,920 farm rent to SNAP income, which pushed gross income to $1,737/mo and net to $1,429/mo, so the household failed the tests. PolicyEngine counts only wages plus financial assistance, giving gross $1,576.90 (121% FPL) and net $1,269.19, below the $1,304.17 net limit. The household is eligible and receives the $23.84/mo one-person minimum allotment ($287.68/yr)." +us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent as SNAP income, which put net income at $1,429 and failed the net test. Without farm rent, net income is $1,269.19, below the $1,304.17 limit. The model also treated '30% of net exceeds the $298 max' as a $0 benefit, but eligible one-person households receive the $23.84/mo minimum allotment, which totals $287.68/yr." +us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"The model included farm rent to reach $1,737/mo gross and declared the household over the gross limit. PolicyEngine's countable gross income is $1,576.90 (121% FPL), which is under both the 130% limit and Texas's 165% BBCE limit. The household is eligible and receives the one-person minimum allotment of $23.84/mo." +us,scenario_112,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"The model ran no computation and defaulted to $0 because it was uncertain about the financial assistance and the vehicle. With $10 in assets, Texas BBCE through TANF non-cash assistance, and net income of $1,269.19 (97% FPL), the household is eligible and receives the $23.84/mo minimum allotment." +us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent to get $1,737/mo and compared it to a 130% gross limit. Countable gross income excludes farm rent and is $1,576.90 (121% FPL), which passes, and the vehicle does not disqualify the household. The benefit is the one-person minimum allotment of $23.84/mo ($287.68/yr)." +us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"The model treated the $13,000 financial assistance as an excluded gift, but PolicyEngine counts it as unearned SNAP income. That raises net income to $1,269.19/mo and the 30% contribution to $380.76, which exceeds the $298 maximum. The household therefore gets only the $23.84/mo minimum allotment, not about $192/mo." +us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"The model left the $13,000 financial assistance out of countable income and included farm rent, which PolicyEngine excludes. Countable income is wages plus financial assistance, so net income is $1,269.19 and the expected contribution of $380.76 exceeds the $298 maximum. The benefit is the $23.84/mo minimum allotment, not $275/mo." +us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"The model assumed the $13,000 financial assistance does not count. PolicyEngine counts it as unearned income, so net income is $1,269.19/mo, not $346. At that income, 30% of net exceeds the $298 maximum and only the $23.84/mo minimum allotment is paid, not $194/mo." +us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"The model counted $1,920 of farm rent to reach $20,843/yr and declared the household over the 130% gross limit. PolicyEngine's countable income is $18,923/yr ($1,576.90/mo, 121% FPL), which passes the gross test and also falls under Texas's 165% BBCE limit. The household is eligible and receives the $23.84/mo minimum allotment." +us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"The model asserted that income was 'far above' the 130% gross limit after counting farm rent. Countable gross income is wages plus financial assistance, $1,576.90/mo (121% FPL), which passes, and net income of $1,269.19 passes the net test. The one-person household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"The model excluded the $13,000 financial assistance from countable income, but PolicyEngine counts it as unearned SNAP income. Net income is then $1,269.19/mo, and the $380.76 expected contribution exceeds the $298 maximum. Only the $23.84/mo minimum allotment ($287.68/yr) is paid, not $194/mo." +us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"The model included farm rent to reach $20,843 and found it above the $20,345 130% gross limit. Countable income excludes farm rent and is $18,923/yr (121% FPL), so the household passes. Net income of $1,269.19/mo passes the net test, and the household gets the $23.84/mo minimum allotment." +us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"The model's $1,737/mo gross figure includes farm rent, which PolicyEngine excludes from SNAP income. Correct gross income is $1,576.90 (121% FPL), which passes the 130% test, and net income of $1,269.19 passes the 100% test. The household receives the $23.84/mo one-person minimum allotment." +us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"The model concluded that because 30% of net income exceeds the maximum allotment, the benefit is $0. That skips the minimum-benefit rule: eligible one- and two-person households get 8% of the one-person maximum ($23.84/mo). The household passes the net test at $1,269.19 once farm rent is excluded." +us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"The model zeroed out the benefit because 30% of net income exceeds the one-person maximum, which ignores the minimum allotment. Eligible one-person households receive 8% of the maximum ($23.84/mo, $287.68/yr). The model's $6,454 annual net figure also does not match the correct $1,269.19/mo net income." +us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"The model's $1,440 net income includes the $1,920/yr farm rent, which PolicyEngine excludes from SNAP income. Excluding it gives net income of $1,269.19, below the $1,304.17 limit. The household is eligible for the $23.84/mo minimum allotment." +us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"The model asserted ineligibility with no computation. Countable income (wages plus financial assistance, excluding farm rent) gives gross $1,576.90 and net $1,269.19, both of which pass. The household receives the $23.84/mo one-person minimum allotment ($287.68/yr)." +us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"The model set SNAP to $0 because 30% of net income exceeds the one-person maximum. That ignores the minimum benefit: eligible one-person households receive 8% of the maximum allotment ($23.84/mo, $287.68/yr)." +us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent, which inflated net income to $1,440, and then reduced the benefit to $0 because 30% of net exceeded the maximum. Without farm rent, net income is $1,269.19 and passes the net test, and one-person households whose contribution exceeds $298 still get the $23.84/mo minimum allotment." +us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"The model gave $0 with no derivation. Countable gross income is $1,576.90 (121% FPL) and net income is $1,269.19 (97% FPL), so the household is eligible. It receives the $23.84/mo minimum allotment." +us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,The model equated a contribution above the maximum allotment with a $0 benefit. It skipped the rule that eligible one-person households receive a minimum allotment of 8% of the maximum ($23.84/mo). +us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"The model claimed income exceeds the net limit, but with farm rent excluded, net income is $1,269.19, below the $1,304.17 limit. The household is eligible, and its allotment is the $23.84/mo minimum, not $0." +us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"The model cited vehicle and asset resources and a contribution above the maximum to reach $0. The $10 in assets passes under Texas BBCE, the vehicle does not disqualify the household, and the one-person minimum allotment of $23.84/mo applies when the contribution exceeds $298." +us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent to reach $1,434 in net income and failed the net test. PolicyEngine excludes farm rent, giving net income of $1,269.19, which passes the $1,304.17 limit. The household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent, used a wrong gross limit (about $17,000 instead of $20,345 at 130% FPL), and treated the $11,838 vehicle as disqualifying. Countable gross income is $18,923/yr, Texas BBCE waives the asset test, and the household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"The model asserted that income and resources were too high without computing them. Countable gross income is $1,576.90 and net income is $1,269.19, both of which pass, and assets are $10. The household gets the $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"The model claimed the facts were insufficient and defaulted to $0. The facts fully determine a one-person household that passes the income tests (net $1,269.19 against a $1,304.17 limit) and is BBCE-eligible, with a $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"The model's $20,843 figure includes farm rent, which PolicyEngine excludes from SNAP income. Countable gross income is $18,923/yr (121% FPL), which passes the gross test, so the household is eligible for the $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"The model found the household over the gross limit after counting farm rent. With farm rent excluded, gross income is $1,576.90/mo (121% FPL), which passes, and the household gets the one-person minimum allotment of $23.84/mo." +us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent and failed the household on the net test. With farm rent excluded, net income is $1,569.90 less deductions, which comes to $1,269.19 and passes the $1,304.17 limit. The household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent and concluded that the household fails the gross test. Countable gross income is wages plus financial assistance, $1,576.90/mo (121% FPL), which passes, so the household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent and concluded that net income exceeds the 100% FPL limit. With farm rent excluded, net income is $1,269.19, below $1,304.17. The household is eligible and receives the $23.84/mo minimum allotment." +us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent, which put net income over the limit. PolicyEngine's net income of $1,269.19 passes the $1,304.17 net limit, and the household receives the one-person minimum allotment of $23.84/mo." +us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"The model concluded that income was 'too high for a positive one-person benefit.' Eligible one-person households always get at least the $23.84/mo minimum allotment, and this household passes the net test at $1,269.19 once farm rent is excluded." +us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"The model said income was too high to produce a benefit, which ignores the minimum allotment. An eligible one-person household whose contribution exceeds the $298 maximum still receives $23.84/mo, and with farm rent excluded, net income of $1,269.19 passes the net test." +us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"The model gave no computation and asserted that limits were exceeded. Countable gross income is $1,576.90 (121% FPL), net income is $1,269.19 (97% FPL), and assets are $10, so the household is eligible for the $23.84/mo minimum allotment." +us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent and concluded that the gross or net limit was exceeded. PolicyEngine's countable income passes both tests, with gross $1,576.90 and net $1,269.19, and the household gets the $23.84/mo minimum allotment." +us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"The model treated the $0 formula result as the benefit and skipped the minimum allotment. Eligible one-person households receive 8% of the one-person maximum ($23.84/mo, $287.68/yr)." +us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"The model included $1,920 of farm rent and failed the 100% FPL net test. With farm rent excluded, net income is $1,269.19, below $1,304.17, and the household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent to get $20,843 against a roughly $20,700 limit. Countable income excludes farm rent and is $18,923/yr (121% FPL), which passes, so the household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"The model's 130% gross-test failure comes from counting farm rent. Countable gross income is $1,576.90/mo (121% FPL), which passes, and the household is eligible for the $23.84/mo minimum allotment." +us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent to reach $20,843 and failed the 130% gross test. Countable gross income is $18,923/yr (121% FPL), so the household passes and receives the $23.84/mo one-person minimum allotment." +us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"The model denied categorical eligibility and let the expected contribution eliminate the allotment. Texas confers BBCE through TANF non-cash assistance, and eligible one-person households receive the $23.84/mo minimum allotment even when 30% of net income exceeds the $298 maximum." +us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"The model invented a rule that a single adult with wages does not qualify for SNAP. One-person households are eligible when they pass the income tests, and this household does (net $1,269.19 against a $1,304.17 limit), so it receives the $23.84/mo minimum allotment." +us,scenario_112,snap,muse-wonder,prompt_ambiguity,categorical_eligibility,False,"The model's $1,737/mo income includes farm rent, which pushed net income over the limit. Excluding farm rent gives net income of $1,269.19, which passes the $1,304.17 limit, and the household gets the $23.84/mo minimum allotment." +us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"The model counted farm rent to get $1,737/mo against a $1,695 limit set at 130% FPL. Countable gross income is $1,576.90 (121% FPL), which passes, so the household receives the $23.84/mo minimum allotment." +us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"The model first derived $0 because 30% of net income exceeded the maximum. It then substituted an unsupported $3,576 by assuming a shelter deduction that the household does not have. The correct result is the one-person minimum allotment of $23.84/mo ($287.68/yr)." +us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"The model asserted ineligibility without computing anything. Countable gross income is $1,576.90 (121% FPL) and net income is $1,269.19 (97% FPL), so the household is eligible and receives the $23.84/mo minimum allotment." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model worked out taxable income of $78,077 and tax of about $11,889, then submitted $8,971 with no supporting calculation. Its itemized deductions also left Virginia income tax out of SALT, left out the $2,434.80 Medicare Part B premium, counted the $2,000 of over-the-counter costs, and skipped the 0.5%-of-AGI charitable floor." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used its own $3,524 Virginia income tax in SALT, while the frozen reference uses a $5,367.30 withholding estimate. Its $4,007 medical deduction counts the $2,000 of over-the-counter costs, which the engine excludes, instead of the $2,434.80 Part B premium the engine imputes. Neither withholding nor the premium is listed in the prompt. These differences put its itemized deductions about $2,278 below the reference's, giving taxable income of $75,085 versus the reference's $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left $19,829 of taxable Social Security out of income and used an invented $28,300 standard deduction. It then added Medicare payroll tax to income tax and applied an arbitrary upward 'adjustment' to reach $10,268, so the number does not come from any valid calculation." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and senior add-on wipe out taxable income. AGI is about $106,573, and even after $33,765 of deductions taxable income is $72,807, which is taxed in the 22% bracket." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Partway through, it reached about $10,711 with the senior deduction, then submitted $5,489 without any calculation behind it. Its itemized total also wrongly counted SALT as the $10,000 cap rather than actual taxes paid, left out Virginia income tax and the Medicare Part B premium, and skipped the 0.5% charitable floor." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It invented $4,500 of mortgage interest from the loan balance even though the prompt gives no interest, left out the $4,106 OBBBA senior deduction, and left Virginia income tax and the Part B premium out of itemized deductions. It then reduced its own $11,802 to $11,210 with no explanation." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the partnership/S-corp income as self-employment income and deducted half of SE tax, and it took the standard deduction, which the recorded audit also gives with no state income tax paid or Part B premium; the frozen reference itemizes $24,787 by adding a $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium. It then cut its own ~$11,900 estimate to $6,900 with no basis." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT of $7,618.50 uses its own Virginia tax of about $3,555, while the frozen reference uses a $5,367.30 withholding estimate. Its $4,007 medical deduction counts the $2,000 of over-the-counter costs the engine excludes, instead of the $2,434.80 Part B premium the engine imputes. Neither withholding nor the premium is listed in the prompt. Itemized deductions are $2,248 below the reference's, and taxable income is $75,055 versus the reference's $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as unchanged TCJA law and left out the $4,106 OBBBA senior deduction. It also invented $6,693 of mortgage interest from the balance, left Virginia income tax out of SALT, skipped the 0.5% charitable floor, and used inflation-guessed brackets instead of the 2026 $12,400/$50,400 thresholds." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518 of itemized deductions left Virginia income tax out of SALT and left out the Medicare Part B premium, and it left out the $4,106 senior deduction entirely. That produced taxable income of $82,183 instead of $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $21,188, below the frozen reference's $24,787. The reference's SALT uses an unlisted $5,367.30 Virginia withholding estimate, and its medical deduction includes an imputed $2,434.80 Part B premium. The model's taxable income is $76,407 versus the reference's $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of SE tax ($1,721) as though the partnership income were self-employment earnings, and left Virginia income tax and the Part B premium out of itemized deductions. It also left out the $4,106 senior deduction, giving taxable income of $82,357." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,106 OBBBA senior deduction and skipped the 0.5% charitable floor. Its SALT uses its own $3,551.53 Virginia tax, while the frozen reference uses an unlisted $5,367.30 withholding estimate. Its $2,007 medical deduction excludes the $2,434.80 Part B premium the engine imputes despite the prompt listing none. Its taxable income is $80,631." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law, with a $5,350 personal exemption and 10%/15%/25% brackets, and dropped the QBI and senior deductions. OBBBA made the TCJA rates permanent, kept the 20% QBI deduction, and added the senior deduction for 2026." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $79,373 leaves out the $4,106 OBBBA senior deduction. Its implied itemized deductions are about $2,460 below the frozen reference's, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium; its explanation does not give the deduction components." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA law: a $5,000 personal exemption and 10%/15%/25% brackets, with no senior deduction. It also subtracted half of SE tax as though the partnership income were self-employment income." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated gross income at $110,094 instead of AGI of $106,573, and left out the QBI and senior deductions, and its listed itemized deductions include no state income tax. The frozen reference's deductions add an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium. The model reports taxable income of about $81,300." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 2026 pre-TCJA sunset law with personal exemptions and the old higher brackets. Under OBBBA, 2026 keeps the TCJA rates and the QBI deduction and adds the senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used TCJA-sunset brackets with a $5,050 personal exemption and no senior deduction, which is not 2026 OBBBA law. Its $19,256 of itemized deductions are below the frozen reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. Under the 2026 brackets, $9,334 implies taxable income of about $66,460, which implies deductions about $6,350 above the frozen reference's and taxable income below its $72,807; the reference depends on unlisted withholding and Part B inputs." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $4,106 OBBBA senior deduction. Its $22,018 of itemized deductions are about $2,770 below the frozen reference's, whose SALT includes an unlisted $5,367.30 Virginia withholding estimate and whose medical deduction includes an imputed $2,434.80 Part B premium. The model's taxable income is $79,682." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction and omitted the $4,106 senior deduction, giving taxable income of about $83,550. The frozen reference's $72,807 uses $24,787 of itemized deductions that include an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $17,518, leaving out Virginia income tax, the Part B premium and the 0.5% charitable floor. It also never applied the $4,872 QBI deduction or the $4,106 senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,It wrongly assumed the QBI deduction expired after 2025 and applied a personal exemption with 10%/15%/25% brackets. OBBBA made the TCJA rates and QBI permanent and added the senior deduction for 2026. +us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 of itemized deductions left out Virginia income tax in SALT and the Medicare Part B premium in medical, and it left out the $4,106 OBBBA senior deduction. That gave taxable income of $84,183 instead of $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard and age-65 deductions eliminate all taxable income. AGI is about $106,573, and the frozen reference's taxable income after deductions including unlisted engine inputs is $72,807, which is taxed in the 22% bracket." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as having modest taxable income, ignoring $86,744 of non-Social Security income plus $19,829 of taxable Social Security. The frozen reference's taxable income, after deductions including unlisted engine inputs, is $72,807; the listed income also contradicts the model's zero-tax explanation." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-2025 sunset law with a personal exemption and the old brackets, and dropped the QBI and senior deductions. Its $17,518 itemized total is below the frozen reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly deducted half of an estimated SE tax on the partnership income and took the standard deduction, while the frozen reference itemizes using an unlisted Virginia withholding estimate and an imputed Part B premium. Its taxable income of $67,020 is about $5,790 below the reference's $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction. Its $11,970 matches the recorded audit's $11,969.80 standard-deduction result with no state income tax paid, no local sales-tax estimate and no Medicare Part B premium. The frozen reference instead counts a $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium, itemizes $24,787 and reaches taxable income of $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $11,540 implies taxable income of about $76,490, about $3,684 above the frozen reference's $72,807. Its explanation gives no deduction figures; if its other deductions match the reference's, its itemized deductions are about $3,684 below the reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $20,639 of itemized deductions and $11,642.20 answer match the recorded $20,639.28 and $11,642.16 alternative with no Part B premium and the engine's $3,654.15 Virginia liability in SALT. The frozen reference instead counts an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium, producing taxable income of $72,807." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $4,106 OBBBA senior deduction, and its implied roughly $21,000 of itemized deductions are below the frozen reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium. That left taxable income at $80,675." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped $10,000 of income, stating AGI as $96,573 instead of $106,573, and took the standard deduction plus the $1,000 non-itemizer charitable deduction while the frozen reference itemizes $24,787 using an unlisted Virginia withholding estimate and an imputed Medicare Part B premium. The wrong AGI also inflated its senior deduction to $4,706." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no itemized figure, and its $11,642.09 answer matches the recorded $11,642.16 alternative with no Part B premium and the engine's $3,654.15 Virginia liability in SALT, where itemized deductions are $20,639.28. The frozen reference instead includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $10,234 implies taxable income of about $70,555, which implies combined deductions about $2,250 above the frozen reference's $33,765 (itemized $24,787 + QBI $4,872 + senior $4,106)." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA sunset law with a $5,468 personal exemption and 10%/15%/25% brackets, and dropped the QBI and senior deductions. OBBBA keeps TCJA rates and QBI for 2026 and adds the senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly assumed the TCJA sunset and QBI expiration, adding a $5,300 personal exemption and the 15%/25% brackets. 2026 law under OBBBA keeps the 12%/22% brackets and the $4,872 QBI deduction and adds the $4,106 senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded the $18,150 standard deduction exceeds itemized deductions. The frozen reference itemizes $24,787 by counting an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium. On its own standard-deduction path, the model omitted the $1,000 non-itemizer charitable deduction, leaving taxable income of $79,445." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $4,106 OBBBA senior deduction, and its SALT used its own $2,746 Virginia tax, while the frozen reference uses a $5,367.30 withholding estimate. Its medical deduction counted over-the-counter costs, which the engine excludes, while the reference includes an imputed $2,434.80 Part B premium; neither withholding nor the premium is listed in the prompt. Its taxable income was $79,437." +us,scenario_114,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets with a $5,300 personal exemption and dropped the senior deduction. It also wrongly deducted half of SE tax on the partnership income, which brought AGI down to $104,852." +us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for federal_income_tax_before_refundable_credits. +us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction because it put itemized deductions at only $17,518. The model gives no itemized breakdown. Its $11,969.85 matches the recorded audit's $11,969.80 standard-deduction result with no state income tax paid, no local sales-tax estimate and no Part B premium. The frozen reference instead counts an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium, itemizes $24,787 and returns $10,729.61." +us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It said taxable income is positive and that it itemizes, yet submitted $0. No nonrefundable credit applies to offset tax on $72,807 of taxable income." +us,scenario_114,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $78,077 is about $5,270 above the frozen reference's $72,807. The reference's itemized deductions include an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium, and exclude the $2,000 of over-the-counter costs; the model gives no itemized breakdown." +us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It put itemized deductions at about $19,500 and chose the standard deduction, while the frozen reference itemizes $24,787 by counting an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium. It also left out the $1,000 non-itemizer charitable deduction from its own standard-deduction path, giving taxable income of $79,445." +us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income, used an outdated $15,750 standard deduction, and skipped itemizing and the senior deduction. Its $15,654 does not even match the tax on its own $66,122 taxable income." +us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It stated AGI as $80,164, losing about $26,400 of income, used a made-up $20,400 standard deduction, and limited QBI to $3,550. It then subtracted the QBI tax effect a second time, ending at $5,290." us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using only $22,700 of wages and ignored the rule requiring the greater of earned income or adjusted gross income. The filer’s other taxable income raises AGI beyond the childless EITC phaseout ceiling, yielding EITC of $0 rather than $295." us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required submission contract." us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly calculated both employee payroll-tax components and correctly excluded self-employment tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its answer effectively duplicates the employee payroll-tax total despite its own reasoning identifying no additional component." @@ -9130,53 +9247,54 @@ us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"Tr us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no self_employment_tax value and no explanation, so there is no substantive calculation to assess. The correct output is $0 because the household has no net earnings from self-employment." us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as SE income and computed $3,442.62. It then submitted an unexplained $3,459.40. The pass-through income is not net earnings from self-employment, so SE tax is $0." us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as net SE income, which is the core error because it is not SE earnings, so SE tax is $0. It also added a made-up 'additional Medicare-equivalent portion above the Social Security wage base'. No income here comes near that base, and this pushed its answer to $3,486.63." -us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model reached about $3,555 using only the $930 personal exemption, so it missed the $800 age-65 exemption. It also itemized $19,518, which skips the 0.5%-of-AGI charitable floor and counts over-the-counter expenses instead of the Medicare Part B premium. It then cut the result to $2,597 with no stated basis." -us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model phased out the age deduction from a $75,000 threshold, which is the married figure. For a single filer the threshold is $50,000, so the deduction is fully phased out and the model wrongly kept $256. Its itemized total of $18,985 applied the charitable floor but left out the $2,434.80 Medicare Part B premium from medical expenses." -us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model left the $19,829 of taxable Social Security out of federal AGI and subtracted charitable gifts above the line. It also miscalculated federal AGI as $27,752 and applied a federal standard deduction. Virginia taxes from the first dollar at 2%, so a positive taxable income cannot produce $0. The correct VA taxable income is $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed the age deduction and a retirement income subtraction would bring taxable income to zero. Virginia has no general pension exclusion, and the $12,000 age deduction is fully phased out because adjusted federal AGI of $86,744 is far above $50,000. That leaves $65,594 taxed at up to 5.75%." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"The model used an $8,750 standard deduction and no exemptions to get $4,227, then submitted an unrelated $1,854. It should have itemized $19,419.94 (charitable after the floor, medical including the Part B premium, and real estate tax) and deducted $1,730 of exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"The model started from an $8,500 standard deduction and $930 exemption, then made a vague, unquantified itemizing adjustment to get $4,014. It never computed the actual $19,419.94 of itemized deductions and left out the $800 age-65 exemption." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model took the $8,500 VA standard deduction even though itemized deductions of $19,419.94 are larger. It also treated the personal exemptions as a credit against tax rather than a $1,730 reduction in taxable income." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model deducted only the $930 personal exemption and missed the $800 exemption for filers aged 65 or older. Its $19,518 itemized total also skips the 0.5%-of-AGI charitable floor and counts over-the-counter expenses as medical in place of the Medicare Part B premium. Together these give taxable income of $66,296 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction instead of itemizing $19,419.94. It then subtracted the $930 and $800 exemptions directly from tax as credits, when they are deductions from taxable income worth only about 5.75% of their face value." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model left taxable Social Security out of federal AGI and then claimed the full $12,000 age deduction as if income were under the limit. Adjusted federal AGI of $86,744 exceeds the $50,000 single threshold by more than $12,000, so the deduction is zero. It also omitted the $1,730 of exemptions and lowered its own $2,948 figure to $2,450 arbitrarily." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518, which leaves the $2,434.80 Medicare Part B premium out of medical expenses and skips the charitable floor. It also deducted only the $930 exemption and missed the $800 age-65 exemption, so taxable income came out at $68,296 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"The model misstated federal AGI as $104,874 and used a made-up $4,800 standard deduction instead of itemizing $19,419.94. That left taxable income at about $78,515 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"The model subtracted the full $12,000 age deduction, but adjusted federal AGI of $86,744 phases it out completely above the $50,000 single threshold. It also took the $8,500 standard deduction instead of itemizing $19,419.94 and omitted the $1,730 of exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $19,518.04 using the full $11,447 of gifts, skipping the 0.5%-of-AGI charitable floor. It also counted the $2,000 of over-the-counter expenses as medical instead of the $2,434.80 Medicare Part B premium. The two errors largely cancel but still overstate deductions by about $98, giving taxable income of $65,496 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model started from federal AGI of $106,573 without subtracting the $19,828.80 of taxable Social Security, which Virginia fully exempts. It also left the Part B premium out of medical expenses and omitted the $800 age-65 exemption." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed $23,948.60 of itemized deductions, about $4,500 more than the allowable $19,419.94. It also deducted only the $800 age exemption and left out the $930 personal exemption." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,180 matches taxable income of about $77,200, which is VA AGI of $86,744 minus roughly $9,550 of standard deduction and exemptions. The model did not itemize the $19,419.94 that the filer's charitable, medical (including Part B), and real estate tax amounts support." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,726 matches taxable income of about $69,280, meaning it deducted only about $17,466 from VA AGI. The full amount is $19,419.94 of itemized deductions plus $1,730 of exemptions, so the model left the Part B premium out of medical expenses and dropped most of the exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized only $15,511 (charitable plus real estate tax) and left out the $4,441.86 medical deduction for expenses above 7.5% of AGI, including the Part B premium. It also used a $1,600 exemption instead of $930 personal plus $800 age-65, which totals $1,730." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $5,617 matches taxable income of about $102,000, which is close to the full federal AGI. The model did not subtract the $19,828.80 of taxable Social Security, the $19,419.94 of itemized deductions, or the $1,730 of exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model figured medical as $10,000 minus 7.5% of AGI, about $2,007, and left out the $2,434.80 Medicare Part B premium that a Medicare enrollee aged 69 pays and can deduct. It also skipped the charitable floor, so itemized deductions came to $17,518 instead of $19,419.94." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,623.52 matches itemized deductions of $17,518. That figure leaves the $2,434.80 Medicare Part B premium out of medical expenses and applies no 0.5%-of-AGI charitable floor, so deductions are about $1,902 too low." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518 and left the $2,434.80 Medicare Part B premium out of the medical deduction, which allows only about $2,007 instead of $4,441.86. It also took no charitable floor, so taxable income came out at $67,496 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"The model laid out the right structure but used a standard deduction instead of itemizing. Its $14,036.62 is impossible under its own bracket schedule, which caps tax on about $77,000 of income near $4,200, so the bracket arithmetic failed outright." -us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model miscalculated VA AGI as $87,744 instead of $86,744 and took an $8,500 standard deduction instead of itemizing $19,419.94. It used $930 for the age exemption instead of $800 and applied a made-up '$270 on the first $8,000' bracket instead of the $720 owed on the first $17,000." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed deductions and exemptions shelter all income. After the Social Security subtraction, $19,419.94 of itemized deductions, and $1,730 of exemptions, $65,593.83 of Virginia taxable income remains and is taxed up to 5.75%." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the retirement income as excluded and the filer as low-income. Virginia taxes the $39,600 of IRA and pension income in full, and the age deduction is fully phased out above $50,000 of adjusted federal AGI, which leaves $65,594 taxable." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518 and left the $2,434.80 Medicare Part B premium out of the medical deduction. It also took no 0.5%-of-AGI charitable floor, so taxable income came out at $67,496 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the Virginia age deduction, but it is fully phased out: adjusted federal AGI of $86,744 exceeds the $50,000 single threshold by more than the $12,000 deduction. That error pushed taxable income down to about $51,647 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model took the $8,750 standard deduction and only the $930 exemption. It should have itemized $19,419.94 and deducted $1,730 of exemptions, including the $800 age-65 exemption." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,658 matches taxable income of about $85,500, which is close to the full VA AGI of $86,744. The model left out the $19,419.94 of itemized deductions and the $1,730 of exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the 0.5%-of-AGI charitable floor correctly but figured medical without the $2,434.80 Medicare Part B premium. As a result its itemized deductions were $16,985 instead of $19,419.94." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,397 matches taxable income of about $63,560, roughly $2,000 below the correct $65,594. That gap fits counting the $2,000 of over-the-counter health expenses as deductible medical expenses, which the medical deduction excludes." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model stated about $57,229 of taxable income, implying roughly $29,500 of deductions and exemptions against VA AGI. The allowable total is $21,150 ($19,419.94 itemized plus $1,730 of exemptions), so it overstated deductions by more than $8,000." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the charitable floor but figured the medical deduction without the $2,434.80 Medicare Part B premium. That gives $16,985 of itemized deductions instead of $19,419.94." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model claimed the standard deduction and exemptions offset all tax. Virginia's deductions total $21,150 against $86,744 of VA AGI, which leaves $65,594 taxable." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $17,518 and left the $2,434.80 Medicare Part B premium out of the medical deduction. It also skipped the 0.5%-of-AGI charitable floor, so deductions were about $1,902 too low." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model itemized $19,518 without the 0.5%-of-AGI charitable floor and counted the $2,000 of over-the-counter expenses as medical instead of the $2,434.80 Medicare Part B premium. Deductions were therefore about $98 too high, and taxable income came out at $65,496 instead of $65,594." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed the filer took the federal standard deduction and so used the $8,500 VA standard deduction. Federal itemized deductions, with real estate and state taxes as SALT plus charitable and medical, exceed the $18,150 federal standard deduction for a single filer aged 65 or older. The filer therefore itemizes in Virginia too, with $19,419.94." -us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"The model subtracted the full $12,000 age deduction, which is fully phased out above $50,000 of adjusted federal AGI. It also left out the $1,730 of exemptions and taxed income over $17,000 at 5.3% instead of Virginia's 5.75% top rate." -us,scenario_114,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"The model deducted only the $930 personal exemption and missed the $800 age-65 exemption. It also mis-subtracted: $86,744 minus $19,647 minus $930 is $66,167, not the $64,446 it stated." -us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_income_tax_before_refundable_credits, so the output was missing." -us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"The model assumed federal non-itemizing and took the $8,500 VA standard deduction instead of itemizing $19,419.94. It also used $930 for the age-65 exemption instead of $800." -us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"The model treated the filer as a low-income retiree with no tax. VA AGI is $86,744, and after $19,419.94 of itemized deductions and $1,730 of exemptions, $65,594 remains taxable at rates up to 5.75%." -us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model correctly treated the age deduction as zero but never subtracted the $19,828.80 of taxable Social Security from federal AGI. It also used a $9,500 standard deduction instead of itemizing $19,419.94 and deducting $1,730 of exemptions." -us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model left Social Security out of AGI and then subtracted the full $23,328 again, excluding it twice. It also invented an SE-tax deduction on partnership income, used a $1,000 standard deduction, and applied the $12,000 age deduction even though it is fully phased out at $86,744 of adjusted federal AGI." -us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model used a wrong federal AGI of $80,164 and applied the $12,000 age deduction, which is fully phased out. It also took a $9,000 standard deduction instead of itemizing $19,419.94 and subtracted $745.50 of nonrefundable credits that do not apply." +us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $19,518, skipping the 0.5%-of-AGI charitable floor and counting $2,000 of OTC spending. The frozen engine instead counts an unlisted $2,434.80 Part B premium and excludes OTC spending, giving $19,419.94 of itemized deductions. It also dropped the $800 aged exemption. Its own steps reached about $3,555, yet it reported an unsupported $2,597." +us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It kept a $256 age deduction by using a $75,000 threshold, but the single threshold is $50,000, so the deduction is fully phased out. Its $18,985 of itemized deductions counts the $2,000 OTC spending in medical costs. The frozen engine excludes OTC spending and instead imputes an unlisted $2,434.80 Part B premium, giving an itemized total about $435 higher." +us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It built a bogus $27,752 AGI by subtracting charitable gifts from income and then applied a federal-style $28,000 standard deduction. It never used VA AGI of $86,744. The frozen engine's itemized deductions and exemptions leave $65,594 taxable, with the medical deduction counting an unlisted Part B premium." +us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed an age deduction and a retirement-income subtraction would wipe out taxable income. The $12,000 age deduction is fully phased out at AFAGI of $86,744, and Virginia has no general pension exclusion, so the income is not wiped out; the frozen engine leaves $65,594 taxable after counting an unlisted Part B premium in itemized deductions." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It took an $8,750 standard deduction and left out the $1,730 of exemptions, which yields $4,227. The frozen engine itemizes $19,419.94 using unlisted inputs; without both its Part B premium and its Virginia withholding estimate in federal SALT, it also takes the $8,750 Virginia standard deduction, yielding $4,127.67 with the exemptions. It then reported an unrelated $1,854." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax with the $8,500 standard deduction and only the $930 personal exemption, which gives $4,188. It then guessed $4,014 without computing the $19,419.94 of itemized deductions or adding the $800 aged exemption." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,500 standard deduction and turned the exemptions into a vague tax reduction. The frozen engine itemizes $19,419.94, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, the engine also takes the standard deduction. It should have subtracted the $1,730 of exemptions from income." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $19,518, skipping the 0.5%-of-AGI charitable floor and counting $2,000 of OTC spending. The frozen engine instead imputes an unlisted $2,434.80 Part B premium and excludes OTC spending, giving $19,419.94 of itemized deductions. It subtracted only the $930 personal exemption and missed the $800 aged exemption, taxing $66,296 compared with the frozen engine's $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $9,000 standard deduction. The frozen engine itemizes $19,419.94, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction. It then subtracted the $930 personal and $800 aged exemptions from the tax as credits, but Virginia deducts them from taxable income." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the full $12,000 age deduction, but AFAGI of $86,744 exceeds the $50,000 single threshold by more than $12,000, so the deduction is zero. It also left out the $1,730 of exemptions and the Part B premium in medical costs, and then cut its own $2,947 result to $2,450 without basis." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 of itemized deductions leaves out the $2,434.80 Part B premium in medical costs and skips the 0.5%-of-AGI charitable floor. It also subtracted only the $930 exemption and missed the $800 aged exemption, so it taxed $68,296." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It misstated federal AGI as $104,874 and used a roughly $4,800 standard deduction plus $800, leaving $78,515 taxable. The frozen engine leaves $65,594 taxable using $19,419.94 of itemized deductions, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the full $12,000 age deduction even though AFAGI of $86,744 phases it out entirely. It also used an $8,500 standard deduction and left out the $1,730 of exemptions. The frozen engine itemizes $19,419.94, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518.04 itemized total skips the 0.5%-of-AGI charitable floor and counts $2,000 of OTC spending. The frozen engine excludes OTC spending and instead imputes an unlisted $2,434.80 Part B premium. These different assumptions put its taxable income at $65,495.96 compared with the frozen engine's $65,593.83, and its tax about $6 below the frozen reference." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It never subtracted the $19,828.80 of federally taxable Social Security, which Virginia excludes. It also missed the $800 aged exemption and taxed $88,125. The frozen engine's $65,594 taxable income includes a medical deduction for an unlisted Part B premium, which the model did not count." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $23,948.60 itemized total exceeds the frozen engine's $19,419.94, which applies the charitable floor and counts an unlisted Part B premium in the medical deduction. It also dropped the $930 personal exemption and kept only $800." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,180 matches about $77,170 of taxable income, consistent with a standard deduction. The frozen engine itemizes $19,419.94 and subtracts $1,730 of exemptions, leaving $65,594 taxable; without both its unlisted Part B premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,726 matches about $69,280 of taxable income, or only about $17,470 of total deductions. That is consistent with leaving the Part B premium out of medical costs and skipping the $1,730 of exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $15,511 itemized total leaves out the $4,441.86 medical deduction, which includes the Part B premium, and skips the charitable floor. It used a $1,600 exemption instead of $930 plus $800." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $5,617 matches about $102,000 of taxable income, so it never subtracted the $19,829 of taxable Social Security. It also applied almost none of the $19,419.94 of itemized deductions or the $1,730 of exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518.04 itemized total includes a medical deduction of about $2,007, matching the engine's $2,007.06 without the unlisted Part B premium. The frozen engine imputes that $2,434.80 premium and deducts $4,441.86 of medical expenses. The model also skips the 0.5%-of-AGI charitable floor, taxing $67,496 compared with the frozen engine's $65,594." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,623.52 matches $17,518 of itemized deductions, which leaves out the $2,434.80 Part B premium and skips the 0.5%-of-AGI charitable floor. That understates deductions by about $1,900 compared with $19,419.94." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $17,518 without the Part B premium or the charitable floor. The frozen engine's $19,419.94 itemized total applies the floor and includes an unlisted $2,434.80 premium. The model taxed $67,496 compared with the frozen engine's $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its $14,036.62 implies roughly $248,000 of taxable income, which is impossible from VA AGI of $86,744. It also used a standard deduction, while the frozen engine itemizes $19,419.94 using an unlisted Part B premium. Without both that premium and its Virginia withholding estimate in federal SALT, the engine also takes the standard deduction. Its bracket arithmetic is broken." +us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,500 standard deduction and set the aged exemption at $930 instead of $800. The frozen engine itemizes $19,419.94 using an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction. It also misstated VA AGI as $87,744 and mangled the bracket arithmetic." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions and exemptions shelter all income. VA AGI after the Social Security subtraction is $86,744. The frozen engine subtracts about $21,150 of itemized deductions and exemptions, including an unlisted Part B premium in medical costs, leaving $65,594 taxable." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed retirement income and age-based relief zero out the tax. Virginia taxes the $30,000 pension and $9,600 IRA, and the age deduction is phased out at AFAGI of $86,744, with the frozen engine leaving $65,594 taxable after counting an unlisted Part B premium in itemized deductions." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 itemized total leaves out the $2,434.80 Part B premium in medical costs and skips the 0.5%-of-AGI charitable floor. That understates deductions compared with $19,419.94, so it taxed $67,496." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $12,000 age deduction on top of itemized deductions, which leaves about $51,647 taxable. AFAGI of $86,744 exceeds the $50,000 threshold by more than $12,000, so the age deduction is zero." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $8,750 standard deduction. Without both the unlisted $2,434.80 Part B premium and $5,367.30 Virginia withholding estimate in federal SALT, the engine does the same, yielding $76,263.77 taxable income and $4,127.67 tax; its $19,419.94 itemized total applies in the frozen calculation that counts the unlisted inputs. It applied only the $930 exemption and missed the $800 aged exemption." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,658 matches about $85,500 of taxable income, so it subtracted almost nothing from VA AGI of $86,744. The frozen engine subtracts $19,419.94 of itemized deductions, including an unlisted Part B premium, and $1,730 of exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $16,985.18 of itemized deductions matches the engine's $16,985.14 without the unlisted Part B premium. Its $3,654.16 answer matches the adjudicated $3,654.15 alternative within a cent; the frozen $3,514.15 reference includes the $2,434.80 premium the engine imputes." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,397 matches about $63,560 of taxable income. That implied taxable income is about $2,037 below the frozen engine's $65,593.83. The engine's $21,149.94 total of deductions and exemptions includes an unlisted Part B premium." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its about $57,229 taxable income matches carrying into Virginia the $96,572.80 federal AGI stated in its federal answer, about $10,000 below the engine's: $76,744 of Virginia AGI less about $17,785 of itemized deductions and $1,730 of exemptions. The frozen engine's $19,419.94 itemized total includes an unlisted Part B premium. Virginia's rates on $57,229 give about $3,033, about $10 below its submitted $3,043.16." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $16,985.18 itemized total matches the engine's $16,985.14 without the unlisted Part B premium. Virginia's rates on its $68,028.82 taxable income give $3,654.16, within a cent of the adjudicated $3,654.15 alternative; its submitted $3,654.66 adds a $0.50 arithmetic slip." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and exemptions offset all tax. VA AGI is $86,744. The frozen engine subtracts about $21,150 of deductions and exemptions, including an unlisted Part B premium in medical costs, leaving $65,594 taxable." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 itemized total omits the Part B premium and skips the 0.5%-of-AGI charitable floor. The frozen engine applies the floor and imputes an unlisted $2,434.80 premium; it leaves $65,593.83 taxable compared with the model's $67,496." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518 itemized total skips the 0.5%-of-AGI charitable floor and counts $2,000 of OTC spending. The frozen engine excludes OTC spending and instead imputes an unlisted $2,434.80 Part B premium. These different assumptions put taxable income at $65,496 compared with the frozen engine's $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the filer takes the federal standard deduction and so used Virginia's $8,500 standard deduction. The frozen engine itemizes $19,419.94 with its unlisted Part B premium. Without both that premium and its $5,367.30 Virginia withholding estimate in federal SALT, it also takes the standard deduction: $8,750 in Virginia, yielding $76,263.77 taxable income and $4,127.67 tax, about $250 less taxable income than the model's $8,500 deduction gives." +us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the fully phased-out $12,000 age deduction and left out the $1,730 of exemptions. It also used a 5.3% top rate instead of Virginia's 5.75%." +us,scenario_114,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the $930 personal exemption and missed the $800 aged exemption. Its stated deductions give $66,167 taxable, not the $64,446 it taxed, so the reported tax rests on a subtraction error." +us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for state_income_tax_before_refundable_credits. +us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,500 standard deduction. The frozen engine itemizes $19,419.94 with its unlisted Part B premium; without both that premium and its $5,367.30 Virginia withholding estimate in federal SALT, it also takes the standard deduction, $8,750 in Virginia, yielding $76,263.77 taxable income and $4,127.67 tax. It also set the aged exemption at $930 instead of $800." +us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as a low-income retiree with no tax. VA AGI is $86,744 after the Social Security subtraction, and the frozen engine's itemized deductions and exemptions leave $65,594 taxable, counting an unlisted Part B premium in medical costs." +us,scenario_114,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It took an $8,500 standard deduction. The frozen engine itemizes $19,419.94 using an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction. It applied only the $930 exemption and missed the $800 aged exemption." +us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It calculated AFAGI net of Social Security but then taxed federal AGI without subtracting the $19,829 of taxable Social Security. It also used a $9,500 standard deduction and left out the $1,730 of exemptions. The frozen engine itemizes $19,419.94 using an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction." +us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It took a half-SE-tax deduction on partnership income, used a nonexistent $1,000 standard deduction and applied the fully phased-out $12,000 age deduction. It also subtracted the full $23,328 of Social Security from an AGI that never included it, and its final $2,453.71 does not match its own $2,542.44." +us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It understated federal AGI at $80,164, used a $9,000 standard deduction (the engine uses $8,750 when neither its unlisted Part B premium nor its Virginia withholding estimate in federal SALT is counted), and applied the fully phased-out $12,000 age deduction. It also invented $745.50 of nonrefundable credits." us,scenario_114,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for the head instead of determining whether PolicyEngine assigned an eligible category. The engine assigns medicaid_category = NONE, so comparing $18,708 of Social Security income with an asserted aged/disabled limit cannot produce eligibility." us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age, disability, low assets, and $1,953 of monthly income as sufficient for an Alabama aged/disabled Medicaid category. PolicyEngine assigns no Medicaid category, so its income-and-assets comparison was applied to a pathway the head does not qualify through." @@ -9189,52 +9307,53 @@ us,scenario_115,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model us,scenario_115,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model waived the gross-income test, excluded tax-exempt interest, and converted a negative allotment formula into an unsupported $142 monthly benefit. Total income of approximately $23,442 exceeds the applicable eligibility limit, and minimum-benefit rules cannot override that ineligibility." us,scenario_115,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly computed that the ordinary allotment formula yields $0 even after excluding tax-exempt interest, but then replaced that result with an unsupported $321 monthly minimum benefit. It also missed that approximately $23,442 of income exceeds the applicable SNAP eligibility threshold; minimum-allotment rules do not grant benefits after failure of eligibility." us,scenario_115,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced all countable income to zero through a standard medical deduction. The listed $250 of annual medical and over-the-counter expenses cannot erase approximately $23,442 of annual household income, which exceeds the applicable one-person SNAP eligibility limit." -us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the $4,734 of federally tax-exempt interest as also exempt in Alabama and got Alabama AGI of $0. Alabama adds this interest back into gross income, so AGI is $4,734. After the $3,000 standard deduction and $1,500 personal exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxable in Alabama, but Alabama includes the $4,734 of federally exempt interest in Alabama AGI. Only $4,500 of deductions offset it ($3,000 standard deduction plus $1,500 exemption), so $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest from Alabama income and assumed 'elderly exemptions' wipe out the liability. Alabama taxes the $4,734 of federally exempt interest and gives no extra age-based deduction. Deductions total only $3,000 plus $1,500, leaving $234 taxable and $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest and assumed income falls below the deduction and exemption threshold. Alabama includes the $4,734 of interest in AGI, and the $3,000 standard deduction plus $1,500 personal exemption total only $4,500, with no age-77 increase. That leaves $234 taxed at 2%, or $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as untaxed in Alabama. Alabama adds federally exempt interest back into its gross income. After the $4,500 of standard deduction and personal exemption, $234 remains and is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model got Alabama AGI of $0 by excluding both Social Security and the tax-exempt interest. Only Social Security is exempt; the $4,734 of federally exempt interest is Alabama income. Subtracting $3,000 plus $1,500 leaves $234 taxable, for $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model claimed Alabama 'taxes no tax-exempt interest'. Alabama includes the $4,734 of federally exempt interest in AGI, and after the $4,500 standard deduction and personal exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model looked only at the Social Security exemption and ignored the $4,734 of tax-exempt interest. Alabama adds that interest to AGI. Subtracting the $3,000 standard deduction and $1,500 exemption leaves $234 taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said tax-exempt interest is not included in Alabama gross income, but Alabama does include federally exempt interest. That makes AGI $4,734, minus $4,500 of standard deduction and personal exemption, leaving $234 taxable and $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated the tax-exempt interest as outside Alabama's tax base. Alabama taxes the $4,734 of federally exempt interest, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model carried federal AGI of $0 straight into Alabama. Alabama adds back the $4,734 of federally tax-exempt interest, so Alabama AGI is $4,734. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest and assumed 'elderly deductions' exceed any income. Alabama includes the $4,734 of interest in AGI and has no age-based standard deduction add-on. The $3,000 standard deduction plus $1,500 exemption leave $234 taxable, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxed in Alabama, but Alabama adds federally exempt interest to gross income. That makes AGI $4,734, minus the $4,500 standard deduction and personal exemption, leaving $234 taxed at 2%, or $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model correctly allowed that the interest might count as Alabama income, then used a nonexistent $4,500 age-65 standard deduction. Alabama's single standard deduction is $3,000 with no elderly add-on, so total deductions are $4,500, not $6,000. That leaves $234 taxable and $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model said Alabama excludes federally tax-exempt interest, but Alabama includes it in gross income. That makes AGI $4,734, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxable at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model counted the $4,734 as income but claimed the standard deduction plus exemption is 'at least $5,000' for a single filer over 65. Alabama gives no age-based increase: the deductions are $3,000 plus $1,500, or $4,500. That leaves $234 taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model assumed taxable income falls below the liability threshold without counting the $4,734 of tax-exempt interest, which Alabama includes in AGI. After only $4,500 of standard deduction and personal exemption, $234 is taxable at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model looked only at the Social Security exemption and left out the $4,734 of federally tax-exempt interest, which Alabama taxes. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model treated tax-exempt interest as excluded in Alabama. Alabama adds the $4,734 of federally exempt interest to AGI, leaving $234 after $4,500 of deductions, which gives $4.68 at 2%." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model asserted income falls below Alabama's threshold without including the $4,734 of tax-exempt interest, which Alabama counts in AGI. Alabama's $3,000 standard deduction and $1,500 exemption cover only $4,500, leaving $234 taxed at 2%, or $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stopped after exempting Social Security and ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said the head has no other taxable income, but Alabama taxes the $4,734 of federally exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxable, which gives $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model exempted Social Security but never added the $4,734 of tax-exempt interest to Alabama AGI as Alabama law requires. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model claimed Alabama exempts federally tax-exempt interest, but Alabama includes it in gross income. That makes AGI $4,734 rather than $0, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said tax-exempt interest is not in the Alabama AGI base, but Alabama adds the $4,734 of federally exempt interest back in. After the $4,500 standard deduction and personal exemption, $234 is taxable at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model treated the tax-exempt interest as non-taxable in Alabama. Alabama includes the $4,734 of federally exempt interest in AGI, which leaves $234 taxable after $4,500 of deductions and gives $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed taxable income is effectively zero without including the $4,734 of tax-exempt interest, which Alabama taxes. Subtracting the $3,000 standard deduction and $1,500 exemption leaves $234 taxed at 2%, or $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model listed no taxable Alabama income, overlooking that Alabama includes the $4,734 of federally tax-exempt interest in AGI. The $3,000 standard deduction and $1,500 exemption cover only $4,500, leaving $234 taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model used zero federal taxable income as the Alabama base. Alabama adds back the $4,734 of federally exempt interest, giving AGI of $4,734. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model said the tax-exempt interest produces no Alabama taxable income, but Alabama includes all $4,734 in AGI. The $3,000 standard deduction plus $1,500 exemption leave $234 taxable, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model concluded the listed facts establish no Alabama-taxable income. The $4,734 of federally tax-exempt interest is Alabama income, and after $4,500 of standard deduction and exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model said no other taxable income was reported, ignoring that Alabama taxes the $4,734 of federally exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model said the tax-exempt interest yields no Alabama taxable income, but Alabama includes the $4,734 in AGI. Deductions of $4,500 leave $234 taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model treated the tax-exempt interest as producing no Alabama taxable income. Alabama adds the $4,734 of federally exempt interest to AGI, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxable, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said there is no taxable income besides Social Security, overlooking that Alabama taxes the $4,734 of federally exempt interest. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model added back the interest but assumed the age-65+ standard deduction and exemptions fully offset $4,734. Alabama's single standard deduction is $3,000 with no age add-on, plus a $1,500 personal exemption, for $4,500 total. That leaves $234 taxable and $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model carried federal AGI of $0 into Alabama, but Alabama adds back the $4,734 of federally exempt interest. The $3,000 standard deduction and $1,500 exemption leave $234 taxable at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model excluded the interest because no out-of-state bond source was identified. Under the modeled rule, all $4,734 of federally tax-exempt interest is Alabama income. After the $4,500 standard deduction and exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model assumed exemptions and deductions would be enough to absorb the interest if it were included. Alabama's deductions total only $4,500 ($3,000 standard plus $1,500 personal exemption), which is less than $4,734. That leaves $234 taxed at 2%, or $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added back the tax-exempt interest but claimed the standard deduction and personal exemption reduce taxable income to $0. They total only $4,500 ($3,000 plus $1,500), so $234 remains taxable at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxable under Alabama law, but Alabama includes the $4,734 of federally exempt interest in AGI. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model added back the interest but used a standard deduction of at least $4,000, for $5,500 total. Alabama's single standard deduction is $3,000 at this income level, so deductions total $4,500. That leaves $234 taxable and $4.68 of tax." -us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Alabama has no state income tax, but Alabama levies a graduated income tax starting at 2%. Its AGI includes the $4,734 of federally exempt interest, and after the $3,000 standard deduction and $1,500 exemption, the tax is $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model said Alabama exempts federally tax-exempt interest, but Alabama includes it in AGI, making AGI $4,734 rather than $0. After $4,500 of deductions, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as not taxable in Alabama, but Alabama includes it in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said taxable income after the standard deduction is $0, either leaving out the $4,734 of interest Alabama counts in AGI or overstating the deductions. Alabama's $3,000 standard deduction plus $1,500 exemption leave $234 taxable, which gives $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Treated the $4,734 of tax-exempt interest as excluded from Alabama income and set Alabama AGI to zero. Alabama adds federally tax-exempt (non-Alabama) interest back into AGI, so AGI is $4,734. Subtracting the $3,000 standard deduction and $1,500 exemption leaves $234, taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Assumed tax-exempt interest is not taxable in Alabama. Alabama AGI includes the $4,734 of federally exempt interest, and after $4,500 in standard deduction plus personal exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Excluded the tax-exempt interest from Alabama income and relied on 'elderly exemptions' that do not exist in Alabama's income tax. Alabama AGI is $4,734. Only the $3,000 single standard deduction and $1,500 personal exemption apply, which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Excluded the tax-exempt interest and assumed a 77-year-old gets larger deduction thresholds. Alabama adds the $4,734 of interest to AGI and has no age-based deduction, so $3,000 + $1,500 = $4,500 in deductions leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Stated that tax-exempt interest is not taxed by Alabama. Alabama includes federally exempt interest from obligations of other states in AGI, so AGI is $4,734. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Set Alabama AGI to $0 by excluding the $4,734 of tax-exempt interest. Alabama adds that interest back, and $4,734 minus the $4,500 standard deduction plus personal exemption leaves $234, taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Asserted that Alabama taxes no tax-exempt interest. Alabama AGI includes the $4,734 of federally exempt interest, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxable at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Correctly exempted Social Security but ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. Deductions of $4,500 leave $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Stated that tax-exempt interest is not included in Alabama gross income. Alabama taxes federally exempt interest from non-Alabama obligations, so AGI is $4,734. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Treated the $4,734 of tax-exempt interest as not subject to Alabama tax. Alabama adds it to AGI, and $4,734 minus $4,500 in standard deduction and exemption leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Started from federal AGI of $0 and never added back the $4,734 of federally tax-exempt interest, which Alabama includes in its AGI. After the $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Excluded the tax-exempt interest and cited 'elderly deductions' that Alabama does not have. Alabama AGI is $4,734. Only the $3,000 standard deduction and $1,500 exemption apply, which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Stated that tax-exempt interest is not taxed in Alabama. Alabama AGI includes the $4,734 of federally exempt interest, and after $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Considered adding the interest back but used a $4,500 'age 65+' standard deduction. Alabama's single standard deduction tops out at $3,000 regardless of age. Total deductions are $3,000 + $1,500 = $4,500, which leaves $234 of the $4,734 taxable at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Excluded federal tax-exempt interest from Alabama income. Alabama includes it, so AGI is $4,734, and the $3,000 standard deduction plus $1,500 personal exemption leave $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Counted the $4,734 of interest but assumed deductions of at least $5,000 for a single filer over 65. Alabama has no age add-on: the $3,000 standard deduction plus $1,500 exemption is $4,500, which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Exempted Social Security but never counted the $4,734 of tax-exempt interest, which is part of Alabama AGI. After $4,500 in deductions, taxable income is $234 and tax is 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Considered only the Social Security exemption and ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Excluded the tax-exempt interest from Alabama taxable income. Alabama adds federally exempt interest to AGI, giving $4,734, and minus $4,500 in deductions that leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Asserted that taxable income is below the threshold without counting the $4,734 of interest in Alabama AGI. That income exceeds the $4,500 standard deduction plus exemption by $234, which is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Exempted Social Security and treated taxable income as zero, omitting the $4,734 of tax-exempt interest that Alabama includes in AGI. After $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Said there was no other taxable income, overlooking that Alabama taxes the $4,734 of federally exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Treated Social Security as the only income and ignored the $4,734 of tax-exempt interest included in Alabama AGI. Deductions of $4,500 leave $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Stated that Alabama exempts federally tax-exempt interest, setting AGI to $0. Alabama includes that interest in AGI ($4,734), and after the $4,500 standard deduction plus exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Left the tax-exempt interest out of the Alabama AGI base. Alabama adds it in, giving AGI of $4,734, and minus the $3,000 standard deduction and $1,500 personal exemption that leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Treated Social Security and tax-exempt interest as producing no Alabama-taxable income. The $4,734 of federally exempt interest is included in Alabama AGI and exceeds the $4,500 in deductions by $234, which is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed taxable income is effectively zero after deductions. Alabama AGI includes the $4,734 of tax-exempt interest, and deductions are only $4,500 ($3,000 standard + $1,500 exemption), which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Found no taxable Alabama income because it excluded the $4,734 of tax-exempt interest. Alabama includes that interest in AGI, and after the $4,500 standard deduction plus exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Started from zero federal taxable income and did not add back the $4,734 of federally exempt interest that Alabama taxes. After $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Concluded that the tax-exempt interest produces no Alabama taxable income. Alabama includes it in AGI ($4,734), and minus the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Excluded Social Security but treated the $4,734 of tax-exempt interest as non-taxable, even though Alabama includes it in AGI. Deductions of $4,500 leave $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Said no other taxable income was reported, missing that Alabama taxes the $4,734 of federally exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Stated that the tax-exempt interest produces no taxable state income. Alabama includes it in AGI, and $4,734 minus $4,500 in deductions leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Treated the reported tax-exempt interest as producing no Alabama taxable income. Alabama adds it to AGI ($4,734), which leaves $234 after the $4,500 standard deduction plus exemption, taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said there was no taxable income other than Social Security, ignoring the $4,734 of tax-exempt interest that Alabama includes in AGI. After $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Claimed the added-back interest is fully offset by deductions for a single filer age 65+. Alabama has no age add-on: $3,000 standard deduction + $1,500 exemption = $4,500 < $4,734, which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Started from federal AGI of $0 without adding back the $4,734 of federally tax-exempt interest that Alabama taxes. After the $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Assumed the tax-exempt interest was Alabama-source because no out-of-state bond amount was identified. The rule here treats all listed federally exempt interest as Alabama income, so AGI is $4,734, and minus $4,500 in deductions that leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Asserted that exemptions and deductions would absorb the interest if it were included. Alabama's $3,000 single standard deduction plus $1,500 exemption total only $4,500, which leaves $234 of the $4,734 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Added back the interest but claimed the standard deduction and personal exemption reduce taxable income to $0. They total only $4,500 ($3,000 + $1,500) against $4,734 of AGI, which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Stated that the $4,734 of tax-exempt interest is not taxable under Alabama law. Alabama includes federally exempt interest in AGI, and after the $4,500 standard deduction plus exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Added back the interest but claimed the standard deduction plus $1,500 exemption is at least $5,500. Alabama's single standard deduction maxes at $3,000, so total deductions are $4,500, which leaves $234 taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Claimed Alabama has no state income tax. Alabama levies a graduated 2%/4%/5% income tax, and its AGI includes the $4,734 of tax-exempt interest. After $4,500 in deductions, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Found no Alabama-taxable income beyond the exempt Social Security, overlooking that Alabama includes the $4,734 of federally exempt interest in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Stated that Alabama exempts federally tax-exempt interest. Alabama AGI includes that $4,734, and after the $4,500 standard deduction plus exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Treated the $4,734 of tax-exempt interest as not taxable in Alabama. Alabama includes it in AGI, and minus the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% = $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed Alabama taxable income is $0 after the standard deduction. Alabama AGI of $4,734 (the tax-exempt interest) exceeds the $3,000 standard deduction plus $1,500 personal exemption by $234, which is taxed at 2% = $4.68." us,scenario_115,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_116,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model improperly awarded a SNAP minimum allotment after its own benefit calculation reduced the regular allotment below zero, even though the household fails the gross-income eligibility test. It also submitted $6,804 despite stating an annual result of $3,402, doubling its own incorrectly calculated amount." us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model itself said the $52,800 in veterans benefits exceeds Florida's disabled-adult income limit, then answered eligible anyway because Florida 'typically provides expanded Medicaid' and has unnamed 'special provisions'. Florida is a non-expansion state with no such override. The spouse gets no SSI and fails the aged/disabled income test, so no category applies." @@ -9251,53 +9370,54 @@ us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categori us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated remaining under age 3 throughout the year as sufficient to qualify as a participant. It omitted the required low-income condition, and this household's approximately $250,733 income is above the qualifying threshold." us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and omitted the income-eligibility test. The household's approximately $250,733 income exceeds the poverty-based Head Start limit, and no other qualifying pathway applies." us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under five as sufficient for WIC eligibility and omitted the separate income test. Although Child 3 meets the categorical age rule, household income exceeds the 185% federal-poverty-guideline threshold for a five-person household, yielding no WIC eligibility." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It phased out the overtime deduction starting at $150,000 AGI, which is the single-filer threshold; the joint threshold is $300,000. That cut the deduction to $15,185 instead of the full $25,000. It also left out the $2,000 non-itemizer charitable deduction and the taxable $2,589 state refund, and it subtracted a $1,200 CDCC even though Head has no earned income, so the CDCC is $0." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced the overtime deduction by 10% of MAGI above $150,000, leaving $14,927. For joint filers the phaseout starts at $300,000, so the full $25,000 is allowed. It also itemized $33,088 instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200). Together these overstated taxable income by $11,185." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of wages, even though wages already include it. It treated charitable gifts and employee business expenses as above-the-line deductions, used an outdated $29,200 standard deduction, and never took the $25,000 qualified-overtime deduction. It also never subtracted the $6,600 nonrefundable CTC, so its figure is essentially a guess at pre-credit tax." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It said tax is 'well above zero' and 'not computed in detail', then submitted 0 as a placeholder. The correct calculation is $191,533.36 of taxable income, $31,561.34 of tax, and $6,600 of CTC, which gives $24,961.34." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed $35,070 of mortgage interest from the loan balance, although no interest paid is listed, so that input is 0. It also claimed a $10,000 SALT amount and itemized $70,970, while leaving out the $25,000 qualified-overtime deduction. On credits, it used 2025 brackets and $2,000 per-child CTC, wrongly gave the 16-year-old only a $500 ODC (a 16-year-old is a qualifying child), and claimed a $600 CDCC that Head's zero earnings rule out." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Its own reasoning reached about $25,972, but it submitted $47,714, which contradicts that work. The reasoning itself assumed mortgage interest from the balance, left out the $25,000 overtime deduction, used $2,000 per-child CTC, and claimed a CDCC that Head's zero earnings block." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed about $80,000 of itemized deductions, including about $32,000 of mortgage interest even though no interest is listed. It deducted the full $35,525 overtime premium instead of capping the deduction at $25,000. It also used $2,000 per-child CTC and a $1,200 CDCC, but the CDCC is $0 because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $33,186 because that beat the $32,200 standard deduction. It missed the 2026 $2,000 non-itemizer charitable deduction for joint filers, which raises the standard path to $34,200. The missed $1,014 of deductions at 22% accounts for its $223 overstatement." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction. It used a $30,000 standard deduction and 2025 brackets in place of the 2026 figures of $32,200 and a 22% bracket ending at $211,400. It also used $2,000 per-child CTC and claimed a $1,200 CDCC, which Head's zero earned income rules out." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reached $22,630, but it submitted $62,700, which has no basis in its own calculation. The reasoning itself assumed $32,000 of mortgage interest that is not in the facts, left out the $25,000 overtime deduction, used $2,000 per-child CTC, and claimed a CDCC that Head's zero earnings block." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It allowed only $12,500 of qualified-overtime deduction, which is the single-filer cap. Joint filers get up to $25,000, and this household is under the $300,000 phaseout. It also itemized about $35,400 instead of taking the $34,200 standard-plus-non-itemizer-charitable path, which left taxable income about $11,300 too high." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed $32,565 of mortgage interest from the loan balance, although no interest is listed, and left out the $25,000 qualified-overtime deduction. It applied only $900 of nonrefundable CTC instead of the full $6,600, which the large tax liability absorbs completely. It also claimed a $1,200 CDCC that Head's zero earned income rules out." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets (15%, 25%, 28%) and treated the CTC as fully phased out, but OBBBA made the TCJA rate structure and the $2,200 CTC with a $400,000 phaseout permanent for 2026. It also left out the $25,000 overtime deduction and itemized instead of taking the $32,200 standard deduction plus the $2,000 charitable deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It removed the entire $35,525 overtime premium from AGI. The qualified-overtime deduction is a below-the-line deduction capped at $25,000. It also itemized with an overstated $3,905 medical deduction and used $2,000 instead of $2,200 per child for the CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, and it used the 2025 $31,500 standard deduction. It also treated only $600 of the CTC as nonrefundable. The full $6,600 CTC offsets regular tax first, and only the unused part goes to the refundable ACTC." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with $5,000 personal exemptions, pre-TCJA rates, and a reduced CTC. OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $2,200 CTC permanent. It also left out the $25,000 overtime deduction and claimed a CDCC that Head's zero earned income blocks." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $46,522 is higher than the tax on the entire $250,733 AGI with no deductions at all. That means it left out the standard deduction, the $25,000 overtime deduction, and the $6,600 CTC. The correct calculation is $191,533.36 taxable income, $31,561.34 of tax, minus $6,600 of CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA 2026 rules, but OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $2,200 CTC permanent. It also left out the $25,000 qualified-overtime deduction and applied a CDCC that is $0 because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired, which led it to use personal exemptions, pre-TCJA brackets, and a $1,000 CTC phased out at this income. Under the permanent OBBBA rules, the $2,200-per-child CTC starts phasing out only at $400,000 for joint filers, so the full $6,600 applies. It also left out the $25,000 overtime deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation, and its $35,283 is about $10,300 above the correct figure. That gap matches leaving out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction and not subtracting the full $6,600 CTC. The correct calculation is $191,533.36 taxable income and $31,561.34 of tax, minus $6,600." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a CTC phaseout, but the $400,000 joint phaseout threshold is far above this household's $250,733 AGI. The full $6,600 CTC therefore applies against $31,561.34 of tax. Its $1,560 overstatement comes from that reduced credit and from understated deductions." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,320 is higher than the tax on standard-deduction-only income, so it left out both the $25,000 qualified-overtime deduction and the $6,600 nonrefundable CTC. The correct calculation is $191,533.36 taxable income and $31,561.34 of tax, minus $6,600." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the taxable $2,589 state refund, so its AGI was $248,145. It also left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, which pushed tax about $8,900 too high." -us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It took only a roughly $31,000 standard deduction and left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction. That left taxable income at $219,734 instead of $191,533. It also used $2,000 per child for the CTC instead of the 2026 amount of $2,200." -us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction. It also treated only $500 per child of the CTC as nonrefundable, but the whole $2,200-per-child credit first offsets regular tax, which is large enough to absorb all $6,600. It also subtracted a $1,200 CDCC, which Head's zero earned income rules out." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the standard deduction and left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, together worth about $5,940 at 22%. It also cited a childcare credit, which is $0 because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed deductions and credits wipe out the liability completely. In fact, taxable income is still $191,533.36 after all deductions, giving $31,561.34 of tax, and the only nonrefundable credit is the $6,600 CTC, which leaves $24,961.34." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $214,996 shows it never took the $25,000 qualified-overtime deduction for the spouse's FLSA overtime premium. It also itemized instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. Those errors left tax $5,234 too high." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its result reconstructs as a $12,500 overtime deduction, which is the single cap instead of the $25,000 joint cap, with no $2,000 non-itemizer charitable deduction. It allowed CTC for only two children, although the children aged 16, 11, and 2 are all under 17, and it subtracted a $1,200 CDCC that Head's zero earned income rules out." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income and $6,600 CTC match the correct calculation. However, it subtracted a $600 CDCC. Under IRC §21(d), creditable expenses are limited to the lower-earning spouse's earned income, and Head's earned income is $0, so the CDCC is $0." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized charitable gifts and Arkansas tax totaling about $33,900. The correct deduction is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200), so its taxable income was about $305 too high." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,080 of charity (after the 0.5% floor) and state tax. It missed that the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction gives $34,200, so taxable income came out about $1,120 too high." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $35,430 of charitable and state-tax deductions, overstating the itemized total. The correct federal deduction path is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200), which gives taxable income of $191,533.36, so its tax came out $271 too low." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $32,200 standard deduction and the $25,000 overtime deduction but left out the 2026 $2,000 non-itemizer cash charitable deduction for joint filers. That left taxable income at $193,534 instead of $191,533.36 and tax $440 too high." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $34,340 of charity and estimated Arkansas tax, overstating the state-tax amount. The correct deduction is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200), so its taxable income was about $139 too low." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave a round guess of $52,000 with no calculation. That figure is higher than the tax on the full AGI with no deductions or credits, so it ignored the $32,200 standard deduction, the $25,000 overtime deduction, the $2,000 charitable deduction, and the $6,600 CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'post-TCJA' sunset law: personal exemptions, a 25% bracket, and a CTC fully phased out. OBBBA made the TCJA brackets permanent and set a $2,200 CTC with a $400,000 joint phaseout. It also left out the $25,000 overtime deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled TCJA expiration, using pre-TCJA 15%/25% brackets, $5,300 exemptions, uncapped SALT, and a $110,000 CTC phaseout. Under OBBBA's permanent rules, the 2026 brackets top out at 22% for this income and the full $6,600 CTC applies. It also left out the $25,000 overtime deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $191,534 and $6,600 CTC are right. However, it computed $31,735 of tax instead of $31,561.48. The 2026 joint brackets give $2,480 + $9,120 + 22% × $90,734 = $31,561.48, so its bracket arithmetic was off by $174." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law (personal exemptions and 15%/25% brackets) and dropped the CTC entirely, and it also subtracted Head's $8,223 premium from the spouse's wages. Under permanent OBBBA law, the $25,000 overtime deduction and the $6,600 CTC apply. It also claimed a $1,200 CDCC, which Head's zero earned income rules out." -us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated itemized deductions at $37,873. It took the full $25,900 of charity without the 2026 0.5%-of-AGI floor, used a $3,116 medical deduction, and included $237 of miscellaneous employee business expenses, which are not deductible. The correct deduction is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200)." -us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to grade." -us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It compared $34,414.79 of itemized deductions against only the $32,200 standard deduction, leaving out the $2,000 non-itemizer charitable deduction that raises the standard path to $34,200. It also overstated the Arkansas income tax part of SALT at $9,027.51. Under the correct calculation the household takes the standard path, with taxable income of $191,533.36." -us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits wipe out all tax, but the only nonrefundable credit is the $6,600 CTC. That covers only part of the $31,561.34 of tax on $191,533.36 of taxable income, leaving $24,961.34." -us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Its own numbers ($31,580 of tax minus $6,600) give about $24,980, but it submitted a rounded $25,000. It also itemized the full $25,900 of charity without the 0.5% AGI floor and compared that total against the $32,200 standard deduction alone. The correct deduction is $32,200 plus the $2,000 non-itemizer charitable deduction ($34,200), giving exactly $24,961.34." -us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed $24,375 of mortgage interest using a fictitious $375,000 debt cap (the acquisition-debt cap is $750,000, and no interest is listed anyway) and itemized $60,275. It left out the $25,000 qualified-overtime deduction, used $2,000 per-child CTC, and claimed a $600 CDCC that Head's zero earned income rules out. These offsetting errors happened to land near the correct figure." -us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the whole $35,525 overtime premium from wages, invented an 'additional standard deduction for net overtime' and an 'overtime deduction credit', used a 32% marginal rate, and applied a nonexistent $409,000 mortgage-debt cap. The actual rule is a below-the-line qualified-overtime deduction capped at $25,000, which leaves taxable income of $191,533.36 in the 22% bracket." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It started the overtime-deduction phaseout at the $150,000 single threshold rather than the $300,000 joint threshold, which cut the deduction to $15,185 instead of the full $25,000. It also claimed a $1,200 CDCC even though Head has $0 earned income, and it left out the $2,000 non-itemizer charitable deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced the overtime deduction by 10% of MAGI above $150,000, leaving $14,927, when joint filers do not start phasing out until $300,000 and get the full $25,000. It also compared its $33,088 of itemized deductions only against $32,200 and ignored the extra $2,000 charitable deduction for non-itemizers, which makes the standard path worth $34,200." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $35,525 overtime premium on top of wages that already include it, and it subtracted charitable gifts above the line. It also used an outdated $29,200 standard deduction and never applied the $25,000 qualified-overtime deduction. Finally, it reported pre-credit tax without subtracting the $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its own reasoning says the tax is well above zero, yet it submitted 0 as a placeholder without doing the calculation. The correct path is taxable income of $191,533, tax of $31,561, minus the $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It made up about $35,070 of mortgage interest from the loan balance even though no interest was listed, and it claimed a $10,000 SALT cap amount with no basis. It also skipped the $25,000 overtime deduction and treated the 16-year-old as a $500 other dependent instead of a $2,200 CTC child. On top of that, it claimed a CDCC even though Head has no earnings, so the near-match is a coincidence of offsetting errors." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning ends near $25,972, but it submitted $47,714, which matches nothing in its own work. The reasoning also made up mortgage interest from the balance, left out the $25,000 overtime deduction, used a $2,000-per-child CTC, and claimed a CDCC that zero Head earnings rules out." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed about $80,000 of itemized deductions by making up mortgage interest and SALT that were never listed, and it subtracted the full $35,525 overtime premium instead of the $25,000 cap. That cut taxable income to about $135,000 instead of $191,533. It also used a $2,000 CTC per child and claimed a CDCC." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized because $33,186 beat the $32,200 standard deduction, but it missed that non-itemizers also get a $2,000 charitable deduction in 2026, so the standard path is worth $34,200. That error added about $1,014 to taxable income, giving $192,548 instead of $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction, and it used 2025 brackets with a $30,000 standard deduction, which left taxable income at $220,734. It also used a $2,000 CTC instead of $2,200 and claimed a $1,200 CDCC even though Head has no earned income." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reaches about $22,630, but it submitted $62,700, which contradicts its own calculation. The reasoning itself made up roughly $32,000 of mortgage interest and $10,000 of SALT, left out the $25,000 overtime deduction, and claimed a CDCC that Head's zero earnings rule out." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped the overtime deduction at $12,500, the single-filer limit, instead of the $25,000 joint limit. It also overstated itemized deductions at about $35,400, which raised taxable income to $202,834 against the correct $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It made up $32,565 of mortgage interest from the loan balance even though no interest was listed, and it left out the $25,000 overtime deduction. It also wrongly limited the nonrefundable CTC to $900, when the full $6,600 offsets tax first, and claimed a $1,200 CDCC despite Head having no earned income." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It used the expired pre-TCJA brackets (15%/25%/28%) and treated the CTC as fully phased out. In fact, 2026 law keeps the 10/12/22% brackets, provides a $2,200 CTC with a $400,000 joint phaseout start, and allows the $25,000 overtime deduction, which it also left out." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It removed the entire $35,525 overtime premium from AGI instead of allowing a below-the-line deduction capped at $25,000. It also overstated medical deductions at $3,905 when the excess over the 7.5% floor is about $741, chose to itemize over the standard plus $2,000 charitable path, and used a $2,000-per-child CTC instead of $2,200." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 overtime deduction and used the 2025 $31,500 standard deduction. It then treated $5,400 of the CTC as refundable and subtracted only $600, but because pre-credit tax exceeds $6,600, the whole CTC is used as a nonrefundable credit." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 rules: $5,000 personal exemptions, the old brackets, and a $1,000 CTC. Current law instead has no exemptions, a $2,200 CTC, a $32,200 standard deduction, and a $25,000 overtime deduction. It also claimed a $1,200 CDCC even though Head has no earned income." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $46,522 is higher than the $37,644 pre-credit tax this household would owe even with only the $32,200 standard deduction and no overtime deduction. So it used higher rates than the 2026 10/12/22/24% brackets and subtracted neither the $25,000 overtime deduction nor the $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired in 2026, so it used pre-TCJA brackets, exemptions, and CTC amounts. Under current law the household gets the $32,200 standard deduction, the $2,000 non-itemizer charitable deduction, the $25,000 overtime deduction, and a $6,600 CTC. It also applied a CDCC that Head's zero earnings rule out." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired, so it used personal exemptions, pre-TCJA brackets, and a $1,000 CTC fully phased out at this income. Under current 2026 law, the $2,200-per-child CTC is not reduced below $400,000 of joint MAGI, and the $25,000 overtime deduction applies." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $35,283 is above the $31,044 the household would owe after the full $6,600 CTC even with no overtime deduction. So it both left out the $25,000 qualified-overtime deduction and failed to subtract the full $6,600 nonrefundable CTC from the $31,561 pre-credit tax." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It cited a CTC phaseout that does not apply below $400,000 of joint MAGI. Its $26,522 answer implies about $7,100 more taxable income than the correct $191,533, which comes from the $32,200 standard deduction, the $2,000 non-itemizer charitable deduction, and the full $25,000 overtime deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,320 is higher than the $37,644 pre-credit tax owed with only the standard deduction. So it applied rates above the 2026 brackets and subtracted neither the $25,000 overtime deduction nor the $6,600 CTC, while the correct result is $31,561 − $6,600." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used an AGI of $248,145, which leaves out the taxable $2,589 state refund. Its $33,816 is higher than the $30,423 that its own AGI yields after the standard deduction and full CTC, so it also skipped the $25,000 overtime deduction and did not subtract the full $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction or the $2,000 non-itemizer charitable deduction, which left taxable income at $219,734 instead of $191,533. It also used a $2,000 CTC per child instead of the 2026 amount of $2,200." +us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 overtime deduction, which left taxable income at about $215,834. It then treated $1,700 per child as refundable and subtracted only $1,500 of CTC, when the full $6,600 is used against the tax first. It also claimed a $1,200 CDCC even though Head has no earned income." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the standard deduction and left out the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction. Its $32,981 is even higher than the $31,044 that the standard deduction alone yields after the full $6,600 CTC, so its credit subtraction was also short." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits wipe out the tax, but pre-credit tax is $31,561 and the only nonrefundable credit is the $6,600 CTC. That leaves $24,961 in liability, not 0." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction for the spouse's $35,525 FLSA overtime premium. That left taxable income at $214,996 instead of $191,533. Its $6,600 CTC and its refusal of the CDCC were both correct." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its answer works out to a $12,500 overtime deduction (the single-filer cap instead of the $25,000 joint cap) plus the $32,200 standard deduction. On top of that, it allowed only two CTC children ($4,400 instead of $6,600) and claimed a $1,200 CDCC despite Head having no earned income. It also left out the $2,000 non-itemizer charitable deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its pre-credit tax of about $31,561 is correct, but it subtracted a $600 CDCC. Under IRC §21(d), qualifying care expenses are capped at the lower-earning spouse's earned income, and Head has $0 and is not a student or disabled, so the CDCC is zero." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,895 of charity and SALT, which is less than the $34,200 that a non-itemizer gets from the $32,200 standard deduction plus the $2,000 charitable deduction. Choosing the smaller itemized total raised taxable income by about $305." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,080 of charity above the 0.5% floor and state income tax, missing that the standard deduction plus the 2026 $2,000 non-itemizer charitable deduction totals $34,200. That raised taxable income by about $1,120." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed about $35,843 of itemized charity and state-tax deductions, which overstates them because it ignored the 0.5%-of-AGI floor on charitable gifts ($1,254). The correct deduction is the $34,200 standard path ($32,200 plus the $2,000 non-itemizer charitable deduction), so it understated taxable income by about $1,640." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $32,200 standard deduction and the $25,000 overtime deduction but left out the $2,000 charitable deduction for non-itemizers that applies from 2026. That raised taxable income to $193,534 instead of $191,533, adding $440 of tax." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $34,249 based on an overstated Arkansas income tax figure. The household's actual itemizable charity, SALT, and medical total falls below $34,200, so the standard deduction plus the $2,000 non-itemizer charitable deduction controls. That gives taxable income of $191,533 rather than its $191,485." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its $52,000 is higher than the $37,644 pre-credit tax owed even with no overtime deduction. So it used inflated rates and subtracted neither the $25,000 overtime deduction nor the $6,600 CTC, while the correct figure is $31,561 − $6,600." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: personal exemptions, a 25% bracket, and a CTC fully phased out. Under current 2026 law, the 10/12/22% brackets apply and the $2,200 CTC is fully available below $400,000. It also left out the $25,000 overtime deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired, using $5,300 exemptions, 15%/25% brackets, and a CTC phased out above $110,000. Current 2026 law instead gives a $32,200 standard deduction, the $25,000 overtime deduction, the 10/12/22% brackets, and a $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $191,534 is correct, but it miscalculated the bracket tax at $31,735. Tax at 10% to $24,800, 12% to $100,800, and 22% above is $2,480 + $9,120 + $19,961 = $31,561, so it overstated the tax by about $174." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired, using personal exemptions, 15%/25% brackets, and no CTC. It also subtracted the ESI premium from wages that were already reported as gross, and it claimed a $1,200 CDCC despite Head having no earned income. In fact, current 2026 law gives a $6,600 CTC and a $25,000 overtime deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $37,873 by overstating medical at $3,116 (the excess over the 7.5% AGI floor is about $741). It also counted $237 of miscellaneous deductions, which are permanently disallowed, and skipped the 0.5% floor on charity. The correct choice is the $34,200 standard path, which gives taxable income of $191,533 instead of its $187,861." +us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or reasoning for this output, so nothing was scored. The correct derivation gives $31,561 in tax on $191,533 of taxable income, minus the $6,600 CTC, which equals $24,961." +us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $34,414.79 based on an Arkansas income tax estimate of $9,027.51, which is too high. It weighed itemizing only against the $32,200 standard deduction and never counted the $2,000 non-itemizer charitable deduction. With both fixed, the $34,200 standard path wins and taxable income is $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits fully offset the tax, but the only nonrefundable credit is the $6,600 CTC against $31,561 of pre-credit tax. That leaves $24,961, not 0." +us,scenario_117,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a $1,200 CDCC even though Head has no earned income, and under IRC §21(d) that caps qualifying expenses at $0. It also left out the $2,000 non-itemizer charitable deduction, so taxable income stayed at $193,534 instead of $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It got close to the correct structure ($25,000 overtime deduction and $6,600 CTC) but miscalculated the bracket tax as about $31,580 and then rounded the result to $25,000. The exact figure is $31,561.34 − $6,600 = $24,961.34." +us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It made up $24,375 of mortgage interest on an assumed $375,000 cap even though no interest was listed (and the joint acquisition-debt limit is $750,000), and it added an unsupported $10,000 SALT amount. It also left out the $25,000 overtime deduction, used a $2,000 CTC per child, and claimed a $600 CDCC even though Head has no earnings." +us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It removed the overtime premium from wages and also invented several items: a $2,800 'additional standard deduction', a $5,400 SALT cap, an overtime 'credit', and a $409,000 mortgage cap penalty. The correct treatment is a $25,000 below-the-line overtime deduction plus the $32,200 standard deduction and the $2,000 non-itemizer charitable deduction, giving taxable income of $191,533." us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It never applied the rule that the ACTC can refund only the CTC left after offsetting tax liability. A federal tax bill of more than $25,000 absorbs all of this household's CTC, so the ACTC is $0. Instead it invented a $3,750 amount 'limited by the ratio of Additional CTC to total CTC'. Along the way it counted the FLSA overtime premium twice on top of gross wages, used the 2023 $27,700 standard deduction and $2,000-per-child CTC, and said income was above the $400,000 phase-out when AGI is about $250,733." us,scenario_117,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"It treated the $1,700-per-child refundable cap as an amount paid out automatically ($5,100). It skipped the step that limits the ACTC to CTC left after tax liability. With more than $25,000 of federal income tax, the full $6,600 CTC is used as a nonrefundable credit, so the refundable portion is $0." us,scenario_117,federal_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"It multiplied 3 children by an $1,800 refundable cap to get $5,400. It never limited the ACTC to CTC left after the credit offsets tax liability. The household's federal tax of more than $25,000 absorbs the full $6,600 CTC, so the ACTC is $0. EITC is also $0." @@ -9333,104 +9453,106 @@ us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The m us,scenario_117,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly computed SS tax of $11,439 at the $184,500 base, Medicare of $3,606 and $0 Additional Medicare Tax. Those sum to $15,045, but it submitted $20,603, a number disconnected from its own components." us,scenario_117,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly transferred a young child's WIC categorical status to the spouse. Living with a WIC-age child does not make an adult eligible; the spouse has no stated pregnancy, postpartum, or breastfeeding status and the household also fails WIC's income test." -us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used a ~$2,340 standard deduction, which put taxable income at $245,805. Itemizing the $24,646.33 charitable deduction (after the 0.5% AGI floor) gives $223,498.20 instead. It then lowered its own ~$9,196 result to $8,081 with an unexplained 'bracket smoothing' step and never applied the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its 2%/4%/3.9% high-income schedule gives $8,586.66 on $222,245, which is about $400 more than the reference's $8,189.53 bracket tax on the larger $223,498.20 base. It also deducted the full $25,900 of charity without the 0.5%-of-AGI floor and subtracted only the $145 personal credits, missing the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages, even though gross wages already include it. It also counted the state refund, which Arkansas excludes, and subtracted made-up self-employment taxes and health premiums. It never itemized the charitable deduction, then applied an outdated 5% top bracket and a made-up $1,050 in credits instead of the $205 actual credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It entered $0 without computing Arkansas tax. The correct computation is $223,498.20 of taxable income (AGI $248,144.53 minus $24,646.33 of charitable itemized deductions), producing $8,189.53 of tax and $7,984.53 after $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It made up ~$22,500 of mortgage interest from the $501,000 loan balance, although unlisted interest paid must be treated as $0. The only itemized deduction is the $24,646.33 charitable amount after the 0.5% AGI floor. It then reported $11,264, which does not match its own ~$7,645 calculation, and it omitted the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $2,589 state refund in Arkansas income, which Arkansas excludes, and made up ~$20,040 of mortgage interest from the loan balance. It then reported $11,820, far above 3.9% of its own ~$202,000 base. It also missed the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It cut taxable income to ~$190,000 by adding mortgage interest and medical costs to the charitable deduction. No mortgage interest is listed, and medical costs are below the floor, so the only itemized deduction is the $24,646.33 charitable amount and taxable income is $223,498.20. It also subtracted only $145 of credits instead of $205." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It kept the $2,589 state refund in income and deducted the full $25,900 without the 0.5%-of-AGI floor, giving taxable income of $224,834 instead of $223,498.20. Its schedule produced $8,685 of tax, well above the reference's $8,189.53. It also omitted the $60 additional credit, subtracting only $145." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $35,525 FLSA overtime premium as an Arkansas exclusion, but Arkansas AGI keeps overtime in wages. It also added a made-up $10,000 SALT deduction and a $448 employee-expense deduction, and claimed a $240 Arkansas child-care credit. That credit is $0 because the head has no earnings, so the federal CDCC earned-income limit is $0. It also missed the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up ~$20,040 of mortgage interest from the $501,000 balance, which is not an interest-paid input, and ignored the 0.5% AGI floor on charitable gifts. It then raised its own ~$8,086 estimate to $12,500 without a basis and never subtracted the $205 of personal and additional credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the Arkansas standard deduction, taxing ~$245,000. Itemizing the $24,646.33 charitable deduction brings taxable income down to $223,498.20. It also left out the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an outdated 4.9% top rate on an understated ~$184k base instead of the 2026 schedule on $223,498.20. It also stated that no nonrefundable credits apply, so it missed the $145 personal credits and the $60 additional credit; the two errors happened to offset near $8,000." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $5,000 standard deduction (taxable income $243,145) instead of itemizing the $24,646.33 charitable deduction, which gives $223,498.20. It also subtracted none of the $205 in nonrefundable credits (5 × $29 personal credits plus the $60 additional credit)." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $185,403.68 taxable income plus $29,805.33 of deductions comes to $215,209. That means it took the $35,525 overtime premium out of the $250,734 federal AGI and kept the state refund, while Arkansas AGI is $248,144.53 with overtime included and the refund excluded. It also over-itemized and applied none of the $205 in nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied an outdated 4.4% top rate with a 2%/4% lower schedule. It also kept the $2,589 refund in income and took a $4,400 standard deduction instead of itemizing the $24,646.33 charitable deduction. It subtracted none of the $205 nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It started from $250,734, keeping the state refund that Arkansas excludes, and deducted the full $25,900 of charity plus $741 of medical costs. Its schedule gave $8,413.63 on $224,093, about $224 more than the reference's $8,189.53 bracket tax. It subtracted only $145, missing the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $10,560 equals exactly 4.4% of $240,000. That matches the outdated 4.4% top rate applied to roughly the full AGI without the $24,646.33 charitable itemization, the 2026 schedule ($8,189.53 on $223,498.20), or the $205 of credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $8,768 equals 3.9% of $224,834 (the $250,734 federal AGI minus $25,900). That is a flat top rate applied to a base that still contains the state refund and skips the 0.5% charitable floor. It ignored the schedule that produces $8,189.53 and subtracted none of the $205 nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $8,623 equals 3.9% of $224,834 minus $145. That means it kept the state refund in income, skipped the 0.5%-of-AGI charitable floor, and taxed everything at the flat top rate instead of the schedule that gives $8,189.53 on $223,498.20. It also left out the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $10,879 matches an outdated ~4.4% rate applied to nearly all of AGI. It skipped the $24,646.33 charitable itemization that reduces taxable income to $223,498.20, ignored the 2026 schedule that produces $8,189.53, and did not subtract the $205 of credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its $8,234.12 sits slightly above the reference's pre-credit tax of $8,189.53. That is consistent with computing the bracket tax on a slightly larger base that keeps the state refund, and then not subtracting the $205 of nonrefundable credits ($145 personal plus $60 additional credit for qualified individuals)." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $9,260 implies taxable income well above $223,498.20. That is consistent with taking a small Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction. It also did not subtract the $205 of personal and additional credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used the Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction. That left roughly $22,000 too much taxable income compared with $223,498.20. It also missed the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied an outdated multi-bracket schedule with 4.7% and 4.4% rates to $222,245 instead of the 2026 schedule, which gives $8,189.53 on $223,498.20. It also deducted the full $25,900 without the 0.5% AGI floor and subtracted none of the $205 nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It stated that no Arkansas nonrefundable credits apply. In fact, $145 of personal credits (5 × $29) plus the $60 additional credit for qualified individuals reduce the $8,189.53 tax to $7,984.53. It also kept the state refund in AGI and added a medical deduction, even though medical costs are below the floor." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It reported only $1,005, as if taxable income were modest. Arkansas taxable income is $223,498.20 (AGI $248,144.53 minus $24,646.33 of charitable itemized deductions), which produces $8,189.53 of tax before $205 of credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and offsets wipe out the tax. The only Arkansas itemized deduction is $24,646.33 of charity, which leaves $223,498.20 taxable and $7,984.53 owed after the $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its high-income schedule gave $8,496.66 on $222,245, more than the reference's $8,189.53 on the larger $223,498.20 base. It also skipped the 0.5%-of-AGI charitable floor and took $160 of credits instead of the $205 actual total ($145 personal plus $60 additional credit)." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the standard deduction instead of itemizing the $24,646.33 charitable deduction, and it stated that no nonrefundable state credit applies. That missed both the $145 personal credits and the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It itemized charity but used a schedule that overstates the bracket tax compared with the reference's $8,189.53 on $223,498.20. It subtracted only the small personal and dependent credits and missed the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It deducted the full $25,900 of charity without the 2026 floor of 0.5% of AGI ($1,253.67), which cut the tax by about $49. It also subtracted only the $145 dependent and personal credits, omitting the $60 additional credit for qualified individuals. Together these leave it $10.78 high." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Its high-income schedule gave $8,578.26 of tax on $222,245, about $390 above the reference's $8,189.53 on $223,498.20. It also skipped the 0.5% charitable floor and subtracted only $145, missing the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $9,535 implies taxable income of roughly $245k or more. That is consistent with using the standard deduction instead of itemizing the $24,646.33 charitable deduction. It also did not subtract the full $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied a ~3.9% top-rate approximation that overstates the bracket tax compared with $8,189.53 on $223,498.20. It also omitted the $60 additional credit for qualified individuals, subtracting only the personal credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Its high-income schedule gave $8,584.76 on $222,245, about $395 more than the reference's bracket tax. It also deducted the full $25,900 without the 0.5% AGI floor and subtracted only $145 of credits, missing the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave a rounded $8,000 guess without computing the steps: Arkansas AGI of $248,144.53, the $24,646.33 charitable itemized deduction, $8,189.53 of bracket tax, and $205 of personal and additional credits." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied an outdated ~4.4% top rate to ~$224,000 instead of the 2026 Arkansas schedule, which gives $8,189.53 on $223,498.20. It also did not subtract the $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"It stated that no nonrefundable Arkansas credits apply, which missed the $145 personal credits and the $60 additional credit for qualified individuals. It also kept the $2,589 refund in AGI and added a $741 medical deduction that does not exist, so its $8,435 exceeds the reference's $8,189.53 pre-credit tax." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It started from a $223,734 AGI, which is $27,000 below the $250,734 federal AGI. That reflects subtracting an overtime-style deduction that Arkansas does not allow, since Arkansas AGI is $248,144.53 with overtime included. The understated base pulled its tax down to $7,491." -us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $242,511 AGI equals $250,734 minus the head's $8,223 premiums, treating them as an above-the-line deduction. They are not: the coverage is employer-sponsored and there is no self-employment income. It also kept the refund in income, skipped the 0.5% charitable floor, applied a flat 3.9% with no lower brackets, and subtracted none of the $205 credits." -us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied a flat ~3.9% to ~$221,000, which overstates the bracket tax compared with the reference schedule's $8,189.53 on $223,498.20. It subtracted only small personal credits and missed the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for state_income_tax_before_refundable_credits. -us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the joint standard deduction, which put taxable income at $245,654, instead of itemizing the $24,646.33 charitable deduction for $223,498.20. It also subtracted only $145 of credits and missed the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported no Arkansas liability. The household has $223,498.20 of Arkansas taxable income, which produces $8,189.53 of tax and $7,984.53 after the $205 of nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It kept the $2,589 refund in AGI, skipped the 0.5% charitable floor, and applied a ~3.8% approximation that overstates the bracket tax. It also used ~$155 of personal credits ($31 × 5) instead of $145 ($29 × 5), and it missed the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 overtime premium on top of gross wages that already include it, and it counted the state refund. It then abandoned its own itemized calculation for a $4,400 standard deduction and applied an outdated flat 4.4% rate, which pushed taxable income to $281,859 instead of $223,498.20." -us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It used a made-up $213,150 income and a $28,867 'filing deduction'. It counted only the $60 additional credit and left out the $145 personal credits. It then submitted $5,105, which does not match its own $7,127 calculation." +us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Took a ~$2,340 standard deduction instead of itemizing the $24,646.33 charitable deduction, so its taxable income was ~$245,805 instead of $223,498.20. It then cut its own $9,196 to $8,081 with an unexplained 'bracket smoothing' step, and it claimed only the $145 personal credits, missing the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Deducted the full $25,900 of charity without the 0.5%-of-AGI floor ($1,253.67), giving $222,245 of taxable income. Its 2%/4%/3.9% schedule produced $8,586.66, well above the engine's $8,189.53 on a larger $223,498.20 base. It also subtracted only $145 of personal credits and left out the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Added the $35,525 overtime premium on top of wages that already include it, counted the state refund, subtracted made-up self-employment taxes, and used a federal-style $258,046 base instead of Arkansas AGI ($248,144.53) less $24,646.33 itemized. It applied a nonexistent 5% top rate and invented $1,050 of credits in place of the actual $205." +us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"Never computed Arkansas tax and entered 0. Arkansas taxes this household's $223,498.20 of taxable income at $8,189.53, less $205 of nonrefundable credits, for $7,984.53." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Made up ~$22,500 of mortgage interest from the $501,000 balance even though no interest paid is listed, which pushed itemized deductions to ~$48,400 instead of $24,646.33. It then submitted $11,264, which does not match its own ~$7,645 working figure." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Counted the $2,589 state refund as Arkansas income and made up ~$20,040 of mortgage interest from the loan balance. Its submitted $11,820 is a ~5.9% effective rate on its own stated ~$202,000 base, which no Arkansas schedule produces. The correct path is $223,498.20 taxable → $8,189.53 − $205." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Added mortgage interest to itemized deductions even though only a mortgage balance is given, which cut taxable income to ~$190,000 instead of $223,498.20 and understated the tax. It also claimed only the $145 personal credits, missing the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Started from $250,734, which includes the $2,589 state refund that Arkansas excludes, and deducted the full $25,900 without the 0.5% AGI floor, ending at $224,834. Its 3.9% top-rate calculation gave $8,685, above the engine's $8,189.53, and it subtracted only $145 of credits, leaving out the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied a nonexistent Arkansas exclusion for the $35,525 FLSA overtime premium and counted the refund. It added a $10,000 SALT deduction with no state or local taxes paid and a $448 employee-expense deduction, so taxable income was $178,861 instead of $223,498.20. It also claimed a $240 Arkansas child-care credit, which is $0 because the head has no earnings and the federal CDCC is therefore zero." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Made up $20,040 of mortgage interest from the $501,000 balance, which brought taxable income down to $202,205. It then dropped its own ~$8,086 result and submitted an unsupported $12,500." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Used the small Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction, taxing ~$245,000 instead of $223,498.20, which overstates the liability by about $800." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Applied a 4.9% top rate that Arkansas no longer has to an understated ~$184k base (the correct base is $223,498.20), and said there are no nonrefundable credits, leaving out the $145 personal and $60 additional credits. Its $8,000 comes from these errors offsetting each other, not from the actual computation." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Used a $5,000 standard deduction instead of itemizing the $24,646.33 charitable deduction, taxing $243,145 instead of $223,498.20. It also applied no nonrefundable credits, leaving out the $205 of personal and additional credits." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its $185,403.68 taxable income equals $250,734 − $35,525 − $29,805.33. That means it excluded the FLSA overtime premium, which Arkansas does not exclude, kept the state refund, and overstated itemized deductions compared with the $24,646.33 charitable-only amount. Its base ended up ~$38,000 below the correct $223,498.20." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied the outdated 4.4% top rate, counted the $2,589 refund in Arkansas income, took a $4,400 standard deduction instead of the $24,646.33 itemized charitable deduction, and applied no nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Started from $250,734, which includes the state refund Arkansas excludes, and itemized the full $25,900 plus $741 of medical without the 0.5% charitable floor. Its bracket tax of $8,413.63 is above the engine's $8,189.53, and it subtracted only $145 of credits, leaving out the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"No reasoning was given. $10,560 matches a rate near 4.4% applied to about $240,000, meaning the outdated top rate on a base that skips the $24,646.33 charitable itemization, and it ignores the $205 of credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"$8,768 equals exactly 3.9% × $224,834, a flat rate on AGI including the refund minus the full $25,900 of charity. That means no lower-bracket schedule, no 0.5% charitable floor, refund exclusion, or subtraction of the $205 nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"$8,623 equals 3.9% × $224,834 − $145: a flat 3.9% on AGI that still includes the $2,589 refund, less the full $25,900 of charity. It skips the actual rate schedule (which gives $8,189.53 on $223,498.20), the 0.5% charitable floor, and the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"No reasoning was given. $10,879 is about 4.4% of ~$247,000, which matches the outdated 4.4% top rate on income reduced only by a small standard deduction, with no charitable itemization and no $205 credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"No reasoning was given. $8,234.12 is above even the engine's pre-credit bracket tax of $8,189.53, so the answer gives no effect to the $205 of nonrefundable credits ($145 personal plus $60 additional) that bring the liability down to $7,984.53." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"No reasoning was given. $9,260 matches taxing roughly the full ~$243k of income after a standard deduction, instead of itemizing the $24,646.33 charitable deduction down to $223,498.20 and subtracting the $205 of credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Stated outright that it used the Arkansas standard deduction, missing that the $24,646.33 charitable itemization is far larger. This overstated taxable income by ~$20,000 and the tax by about $1,200." +us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Used an outdated bracket table with 3.4%/4.7%/4.4% rates instead of the 2026 schedule, deducted the full $25,900 without the 0.5% AGI floor, and subtracted no nonrefundable credits, leaving out the $145 personal and $60 additional credits." +us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"Counted the $2,589 refund in Arkansas income, added a medical deduction that is actually under the floor, and skipped the 0.5% charitable floor. It then said no Arkansas nonrefundable credits apply, leaving out the $145 personal and $60 additional credits, and rounded to $8,000." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Treated the liability as 'modest' and gave $1,005, about one-eighth of the correct tax. Arkansas AGI is $248,144.53 with only $24,646.33 itemized, so the schedule gives $8,189.53 before $205 of credits; the answer amounts to taxing only a small slice of income." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Claimed that deductions and offsets wipe out the Arkansas tax. In fact itemized deductions are only $24,646.33 against $248,144.53 of Arkansas AGI, leaving $223,498.20 taxable and $7,984.53 owed after credits." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Deducted the full $25,900 without the 0.5% AGI floor (taxable $222,245). Its high-income schedule produced $8,496.66, above the engine's $8,189.53 on the larger correct base, and it took $160 of credits instead of $145 personal plus the $60 additional credit ($205)." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Used the standard deduction instead of itemizing the $24,646.33 charitable deduction and said no nonrefundable credit applies, leaving out the $145 personal and $60 additional credits." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $8,107 after only small personal/dependent credits means a bracket tax of about $8,250, above the engine's $8,189.53 on $223,498.20. It also left out the $60 additional credit for qualified individuals that brings total credits to $205." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"$7,995.31 matches deducting the full $25,900 without the 0.5%-of-AGI floor ($1,253.67) while subtracting only the $145 personal credits and dropping the $60 additional credit. The two errors nearly cancel, leaving it $10.78 above the correct $223,498.20 → $8,189.53 − $205 path." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Deducted the full $25,900 without the 0.5% floor (taxable $222,245). Its high-income schedule gave $8,578.26, well above the engine's $8,189.53 on $223,498.20, and it subtracted only $145 of credits, leaving out the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"$9,535 matches taxing essentially all wage income after only a standard deduction, without the $24,646.33 charitable itemization that brings taxable income down to $223,498.20, and without the full $205 of credits." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Applied a 3.9% marginal rate to its post-charity base, which overstates bracket tax compared with the engine's 2026 schedule ($8,189.53 on $223,498.20, equal to 3.7% minus $79.90). It also counted only personal credits and left out the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Deducted the full $25,900 without the 0.5% AGI floor and used a 3.9% high-income schedule, which gives about $8,585 instead of the engine's $8,189.53. It subtracted only $145 of personal credits and left out the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Gave a rounded $8,000 guess with no computation. The correct path is $248,144.53 AGI − $24,646.33 itemized = $223,498.20, taxed at $8,189.53, minus $205 of nonrefundable credits = $7,984.53." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Applied the outdated ~4.4% top rate to ~$224,000 instead of the 2026 schedule that gives $8,189.53 on $223,498.20, and subtracted none of the $205 of personal and additional credits." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Started from federal AGI $250,734, which includes the refund Arkansas excludes, added $741 of medical, and skipped the 0.5% charitable floor. It used a 3.9% top rate and said no nonrefundable credits apply, leaving out the $145 personal and $60 additional credits." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Started from an unsupported $223,734 'federal AGI', $24,411 below the correct Arkansas AGI of $248,144.53, so taxable income came to $197,834 instead of $223,498.20. It also subtracted only the $145 personal credits, missing the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $242,511 AGI equals $250,734 − $8,223: it kept the state refund and subtracted the head's employer-plan premiums as an above-the-line deduction, which Arkansas does not allow. It then applied a flat 3.9% with no bracket schedule and no 0.5% charitable floor, and subtracted none of the $205 of credits." +us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Applied about 3.9% to ~$221,000, which overstates bracket tax compared with the engine's $8,189.53 on $223,498.20. It subtracted only small personal credits and left out the $60 additional credit, which accounts for most of its ~$515 overstatement." +us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer and no reasoning for state_income_tax_before_refundable_credits, so there is nothing to score." +us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Used the joint standard deduction (taxable $245,654) instead of itemizing the $24,646.33 charitable deduction, and subtracted only $145 of personal credits, missing the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Said there is no Arkansas liability, but $223,498.20 of Arkansas taxable income produces $8,189.53 of tax, and only $205 of nonrefundable credits apply, leaving $7,984.53." +us,scenario_117,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Correctly removed the state refund but used the joint standard deduction (taxable $243,435) instead of itemizing the $24,646.33 charitable deduction. It also subtracted only $145 of personal credits, missing the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Kept the $2,589 refund in Arkansas AGI ($250,734 instead of $248,144.53) and deducted the full $25,900 without the 0.5% floor. It applied a ~3.8% rate that gave $8,450 instead of $8,189.53, and it used a made-up $31 × 5 = $155 credit instead of $145 personal plus the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Added the $35,525 overtime premium on top of wages that already include it and counted the refund, reaching $286,259. It then took a $4,400 standard deduction and a flat outdated 4.4% rate, throwing out its own itemized calculation, instead of taxing $223,498.20 under the 2026 schedule." +us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Its $213,150 starting income equals $248,675 − $35,525, so it excluded the FLSA overtime premium, which Arkansas does not exclude, and then subtracted an invented $28,867 deduction instead of the $24,646.33 itemized amount. It applied only the $60 additional credit and missed the $145 personal credits, and its final $5,105 does not match its own $7,127 calculation." us,scenario_117,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_118,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model said Social Security income is modest but still concluded 'no Medicaid enrollment assumed; not eligible'. That confuses enrollment with eligibility, and Medicare eligibility does not rule out Medicaid. It also missed that $2,800 a year in Social Security and $70 in assets qualify the head for SSI ($9,368 a year), and in NY that gives automatic Medicaid through the SSI-recipient category." us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said the facts were not enough to support eligibility, even though they fully establish the aged/blind/disabled pathway. The head is 74, blind, and disabled, with $2,800 a year in Social Security and $70 in assets. That qualifies them for SSI, which gives automatic NY Medicaid through the SSI-recipient category, and they also pass NY's aged/disabled income and asset tests. The model never applied these tests to the listed income and resources." us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the $2,800 Social Security retirement benefit is not subject to employee payroll tax and that no wages were provided, but then contradicted that reasoning by assigning $172.90 of employee-side tax. It improperly placed retirement benefits or another nonexistent amount in the payroll-tax base instead of applying the tax rates to a zero earned-income base." -us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $233/month of Social Security and left out the $780.67/month of SSI the head receives, so it understated SNAP gross income, which is $1,014. With that low income and an assumed $857 utility allowance, net income went to $0 and it paid a $293 maximum. The correct net income is $194.38, which gives $239.80/month for most of the year." -us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted SS plus SSI ($1,014) and the $209 standard deduction. But it built shelter costs from mortgage interest and property tax alone ($599) and left out the utility allowance PolicyEngine adds, which brings shelter costs to about $1,013. That left net income at $609 instead of $194.38. It also missed the October–December FY2027 increase to $248.58/month." -us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Concluded the household exceeds SNAP income limits even though its own reasoning describes very low income and large shelter deductions. The household is categorically eligible through SSI and easily passes the net income test ($194.38 against $1,304 FPG), so its $0 contradicts the $239.80/month allotment." -us,scenario_118,snap,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 without doing a calculation. It missed that this one-person elderly/disabled household is categorically eligible through SSI. With $1,014 gross income and the standard and uncapped shelter deductions, it gets $239.80–$248.58/month." -us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Treated $233/month of Social Security as the only income and left out the $780.67/month of SSI that SNAP counts as unearned income. That drove net income to $0 and the benefit to a $293 maximum. With SSI counted, net income is $194.38 and the allotment is $239.80/month." -us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI and ignored the listed shelter costs, so it computed net income of only $29. Its final $2,533 also doesn't match its own figure of about $283/month (about $3,400/year)." -us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only Social Security, leaving out the $780.67/month of SSI, and skipped the shelter deduction. It also made an arithmetic error: $298 − 0.3 × $29 is $289, not the $203 it used." -us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SS plus SSI ($1,014) and the $209 standard deduction correctly. But it limited shelter costs to the $599 of mortgage interest and property tax, without the utility allowance PolicyEngine includes. The result was net income of $609 instead of $194.38 and a $115 benefit instead of $239.80." -us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly assumed the head gets no SSI. In fact $233/month of Social Security against the $994 SSI benefit rate leaves an SSI payment of $780.67/month, which SNAP counts as income. Leaving it out zeroed net income, and it then used the FY2025 $292 maximum instead of $298." -us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Left the $780.67/month of SSI out of gross income and concluded that net income was near zero. Its unexplained reduction from a $3,504 maximum to $2,555 doesn't follow from any benefit-reduction calculation." -us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted income as only $233/month, leaving out the SSI that brings SNAP gross income to $1,014. As a result net income fell to $0 and it gave the $298 maximum. Correct net income is $194.38, which leaves a $58.20 expected contribution." -us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,Left out the $780.67/month of SSI from countable income and also skipped the excess shelter deduction for the listed mortgage interest and property tax. It used a $199 standard deduction instead of $209 and ended up near the maximum benefit. -us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Did not count the SSI payment ($780.67/month) as SNAP income, so net income came out to $0. It then applied the FY2025 $292 maximum, which overstates the benefit compared with the $239.80/month that follows from $194.38 net income." -us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI, but it computed the excess shelter deduction ($198.75) from mortgage interest and property tax alone, without the utility allowance. That left net income at $601 instead of $194.38. It also used a $303 maximum and a $210 standard deduction instead of $298 and $209." -us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly excluded SSI from SNAP income, but SSI is countable unearned income under 7 CFR 273.9(b)(2). Leaving out $780.67/month drove net income to zero, and it then used a $299 maximum allotment." -us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $864.67 (the correct amount is $780.67). It also assumed a utility allowance large enough to wipe out all net income. PolicyEngine's total shelter costs of about $1,013 leave $194.38 of net income, and the correct maximum is $298, not $292." -us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,Returned $0 without a benefit calculation. It missed that the household is categorically eligible through SSI and that $194.38 of net income yields $239.80/month. -us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $817 (correct: $780.67) and used outdated values: a $198 standard deduction instead of $209 and a $291 maximum instead of $298. It limited shelter costs to mortgage interest and property tax, so net income came out at $679 instead of $194.38." -us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the NYSNIP pathway grants the maximum benefit automatically. PolicyEngine applies the regular SNAP formula, which subtracts 30% of the $194.38 net income (including SSI) from the $298 maximum. It also used a stale $291 maximum." -us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Asserted a $0 benefit with no calculation and counted only the $2,800 of Social Security. This one-person SSI recipient has net income of $194.38 and gets $239.80–$248.58/month." -us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only Social Security, leaving out the $780.67/month of SSI. That zeroed net income, and it then applied a stale $291 maximum. With SSI counted, net income is $194.38 and the benefit is $239.80/month." -us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $828/year ($69/month) implies net income of about $760/month, which means it took almost no excess shelter deduction against the $805 of adjusted income. PolicyEngine's shelter deduction brings net income down to $194.38." -us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Understated SSI at $9,044 (correct: $9,368) and used outdated values: a $200/month standard deduction instead of $209 and a $292 maximum instead of $298. It also computed excess shelter from mortgage interest and property tax only, without the utility allowance, so net income came out at $582/month instead of $194.38." -us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Started from $233.33/month of Social Security and left out the $780.67/month of SSI SNAP counts. Net income therefore became $0, and it used a guessed $295 maximum instead of $298 minus a $58.20 contribution." -us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the SSI income and also ignored the listed mortgage interest and property tax shelter costs. It reduced the maximum by 30% of only $26 of net income, using guessed values ($207 standard deduction, $296 maximum)." -us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed no expenses were listed, although mortgage interest and property tax are, and returned $0. The household is categorically eligible through SSI and, after the standard and uncapped shelter deductions, has $194.38 of net income, which yields $239.80/month." -us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 on the grounds that the facts were insufficient. The listed income, assets, and shelter costs are enough to compute a positive SNAP allotment for this SSI-eligible elderly/disabled household." -us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the SSI program's $20 general income exclusion as if it were a SNAP deduction and left SSI out of SNAP income, so net income came out to zero. It also used a $243 maximum allotment instead of $298." -us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $2,800 of Social Security and only the standard deduction, leaving out SSI income and the shelter deduction. The result was a near-maximum $290.70/month instead of $239.80." -us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SS plus SSI correctly, but it built the excess shelter deduction from mortgage interest and property tax alone, without the utility allowance PolicyEngine adds. That left net income near $609 instead of $194.38 and produced $115/month instead of $239.80." -us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SS and SSI but limited shelter costs to the listed mortgage interest and property tax, without the utility allowance. Its $123/month implies net income near $583, against PolicyEngine's $194.38." -us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted $1,014 of SS plus SSI and the $209 standard deduction. But it explicitly took no shelter deduction, even though mortgage interest and property tax are listed and the elderly/disabled excess shelter deduction is uncapped. That left net income at $805 instead of $194.38." -us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Treated income as very low by leaving out the $780.67/month of SSI, so it assigned the $298 maximum. With SSI counted, net income is $194.38 and the 30% contribution is $58.20/month." -us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $10,412/year (correct: $9,368) and computed only a $1,833 excess shelter deduction from mortgage interest and property tax, without the utility allowance. That gave net income of $739/month instead of $194.38." -us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used the right $298 maximum, $209 standard deduction, and SS plus SSI income. But its shelter costs were only mortgage interest and property tax ($598.75), without the utility allowance. That gave net income of $608.75 instead of $194.38." -us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 on the claim that the inputs were missing. It never applied SSI categorical eligibility or the SNAP formula, which gives $239.80/month for January–September and $248.58/month for October–December." -us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,Identified SSI categorical eligibility but estimated net income at about $735/month. That is far above the $194.38 that results from adding the utility allowance to mortgage interest and property tax in the uncapped shelter deduction. It also used a $301 maximum instead of $298. -us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left the SSI payment out of countable income, so net income became $0. It then applied the FY2025 $292 maximum instead of $298 minus the $58.20 contribution." -us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted SS of $2,800 plus SSI of $9,368 and the $209 standard deduction. But it ran the uncapped excess shelter calculation on only $7,185 of mortgage interest and property tax, without the utility allowance. That gave net income of $608.75/month instead of $194.38." -us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed shelter costs were unlisted, although mortgage interest and property tax are given. It also left out the $780.67/month of SSI and used 2024 values ($193 standard deduction, $291 maximum)." -us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,Counted only the $233/month of Social Security and left out the $780.67/month of SSI. That drove net income to $0 and gave the $298 maximum instead of $239.80/month. -us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left the SSI payment out of SNAP gross income, so net income came out to $0. It then used an overstated $309 maximum instead of $298." -us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $10,412/year (correct: $9,368). It also claimed no housing input was supplied and skipped the excess shelter deduction, even though mortgage interest and property tax are listed. That inflated net income to $892/month." -us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Reasoned that income was 'too low to qualify', which inverts the SNAP formula: lower net income raises the allotment. This SSI-eligible household gets $298 minus 30% of $194.38 of net income." -us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Counted $1,014 of income and the $209 deduction correctly. But it set shelter costs at only the $598.75 of mortgage interest and property tax, without the utility allowance PolicyEngine adds. That gave an excess shelter deduction of $196.25 instead of about $611 and net income of $608.75 instead of $194.38." -us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,Left out the $780.67/month of SSI and never applied the shelter deduction for the listed mortgage interest and property tax. It reduced only a stale $291 maximum by 30% of $35. -us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Treated SNAP as requiring a qualifying child, but single-adult households are fully eligible. This elderly/disabled SSI recipient qualifies for $239.80–$248.58/month." +us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $233/month of Social Security and left out the $780.67/month of SSI the blind/disabled head receives, which SNAP counts as income (gross $1,014). With SSI counted, the shelter deduction leaves $194 of net income and a $58.20 contribution, not $0. Its $293×12 maximum-allotment answer therefore overstates the benefit." +us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI correctly ($1,014 gross) and applied the uncapped shelter deduction, but it used only mortgage interest plus property tax ($599/month) as shelter costs and left out the New York standard utility allowance. The SUA-inclusive excess shelter deduction is about $611/month, not $196. That gives $194 of net income instead of $609, and $239.80/month (rising to $248.58 in October–December) instead of $115." +us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It ruled the household ineligible even though gross income is 78% of FPG, assets are $70, and SSI receipt confers categorical eligibility. It never ran the benefit formula, which gives the $298 maximum minus a $58.20 contribution, or $239.80/month." +us,scenario_118,snap,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without doing any calculation, ignoring that the household passes every income and asset test and is categorically eligible through SSI. The formula gives $239.80/month for January–September and $248.58 for October–December, or $2,903.94 a year." +us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $233 of Social Security as the only income and left out the $780.67/month of SSI, which drove net income to $0 and the answer to the ~$293 maximum allotment. It also explicitly declined the standard utility allowance, which PolicyEngine includes in shelter costs. Counting SSI and the SUA-inclusive shelter deduction gives $194 of net income and a $58.20 contribution." +us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It left SSI out of income and ignored the shelter deduction, then submitted $2,533, which matches none of its own intermediate figures (~$3,400). Counting the $780.67 of SSI and subtracting the $209 standard deduction and the ~$611 SUA-inclusive excess shelter deduction gives $194 of net income and $239.80/month." +us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left out SSI and the shelter deduction, which gave net income of $29, and then misapplied its own formula: $298 − 0.3×$29 is $289, not $203. The correct path counts $1,014 of gross income (SS plus SSI) and subtracts $209 and ~$611 in deductions, leaving $194 of net income and a $239.80 benefit." +us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI ($1,014 gross) but built shelter costs from mortgage interest and property tax alone ($599) and left out the New York standard utility allowance. That understated the excess shelter deduction ($196 instead of ~$611) and overstated net income ($609 instead of $194). It also took 12 flat months, missing the higher $248.58 benefit for October–December." +us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly assumed the head gets no SSI, but a blind/disabled person with $2,800 of SS receives $780.67/month of SSI, which counts as SNAP income. Leaving it out drove net income to $0 and the answer to the maximum allotment, and it used the FY2025 $292 maximum instead of $298." +us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left SSI out of income, concluded net income was near zero, then cut the $3,504 maximum to $2,555 without computing any net income. Counting the $780.67 of SSI and the ~$611 SUA-inclusive shelter deduction gives $194 of net income, a $58.20 contribution, and $239.80/month." +us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $233/month of Social Security and left out the $780.67/month of SSI, so net income fell to $0 and it returned the $298×12 maximum. With SSI counted, net income is $194 and the $58.20 contribution brings the benefit down to $239.80/month." +us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It left out both SSI and the excess shelter deduction, subtracting only a $199 standard deduction from $233 of SS. The correct base is $1,014 of gross income (SS plus $780.67 SSI) less $209 and a ~$611 excess shelter deduction, which leaves $194 of net income and a $58.20 contribution." +us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It reached $0 net income only by leaving out the $780.67/month of SSI, and it used the FY2025 $292 maximum allotment. With SSI counted, net income is $194, so the benefit is $298 − $58.20 = $239.80/month, not the maximum." +us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,It counted SSI but built shelter costs from mortgage interest and property tax only and left out the New York standard utility allowance. That gave a $198.75 excess shelter deduction instead of ~$611 and net income of $601 instead of $194. It also used a $303 maximum and a $210 standard deduction instead of $298 and $209. +us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly excluded SSI from SNAP income, but SSI is countable unearned income for SNAP, so gross income is $1,014, not $233. With SSI counted, net income is $194 rather than $0 and the benefit is $239.80/month, not a $299 maximum." +us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI (overstated at $864.67 instead of $780.67) and included the utility allowance, but asserted $0 net income without computing it. Adjusted income of $805 less the ~$611 excess shelter deduction leaves $194 of net income and a $58.20 contribution. It also used $292 rather than $298 as the maximum." +us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 on the grounds that income was too low to generate a benefit, which inverts the SNAP formula: lower net income means a larger allotment. This categorically eligible SSI household receives $298 − $58.20 = $239.80/month." +us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated SSI at $817/month, used a $198 standard deduction and a $291 maximum, and built shelter costs from mortgage interest and property tax without the New York standard utility allowance. That produced a $173 excess shelter deduction and $679 of net income, instead of ~$611 and $194." +us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It replaced the regular SNAP formula with a standardized maximum benefit under the NY Nutrition Improvement Project for SSI recipients. PolicyEngine computes the regular formula, where $194 of net income produces a $58.20 contribution and a $239.80 monthly benefit. It also used $291 rather than $298 as the maximum allotment." +us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 even though the household passes the gross, net, and asset tests and is categorically eligible through SSI. The formula gives $298 − $58.20 = $239.80/month, or $2,903.94 a year once the higher October–December months are included." +us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $291×12 answer implies $0 net income, which only follows if the $780.67/month of SSI is left out of income. With SSI counted, net income after the standard and SUA-inclusive shelter deductions is $194, and the benefit is $239.80/month against a $298 maximum." +us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $828 ($69/month) implies about $763 of net income against the $298 maximum. That fits counting SSI while taking almost none of the excess shelter deduction. The correct deduction is ~$611 once mortgage interest, property tax, and the NY utility allowance are included, which leaves $194 of net income and a $239.80 benefit." +us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It understated SSI at $9,044/yr instead of $9,368, used a $200/month standard deduction and a $292 maximum, and limited shelter costs to mortgage interest and property tax without the New York standard utility allowance. That gave a $2,463/yr excess shelter deduction instead of ~$7,327 and $6,981 of annual net income instead of about $2,333." +us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233 of Social Security as the only income and left out the $780.67/month of SSI, which drove net income to $0 and the answer to an estimated $295 maximum. With SSI counted, net income is $194 and the benefit is $298 − $58.20 = $239.80/month." +us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out both SSI and the excess shelter deduction, subtracting only a ~$207 standard deduction from $233 of SS. The correct net income is $194: $1,014 of gross income (SS plus SSI) less $209 and a ~$611 shelter deduction, which gives a $58.20 contribution." +us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the household had no expenses or eligibility facts, ignoring the listed mortgage interest and property tax and the SSI the blind/disabled head receives. The household is categorically eligible through SSI and gets $298 − $58.20 = $239.80/month." +us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 for lack of 'supporting facts', but low-income, low-asset elderly/disabled households qualify for SNAP without any further facts, and SSI receipt confers categorical eligibility. Running the formula gives $239.80/month for January–September and $248.58 for October–December." +us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $780.67/month of SSI out of income (and applied the SSI program's $20 exclusion, which SNAP does not use), so net income fell to $0. It then used a $243 maximum allotment instead of the FY2026 $298. These two errors offset each other, putting its answer close to the formula result of $298 − $58.20 = $239.80/month." +us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,488.40 equals 12×($298 − 0.3×$24.33), which means it subtracted only the $209 standard deduction from $233.33 of SS. It left out both the $780.67/month of SSI and the excess shelter deduction, which together give $194 of net income." +us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI, but its $115/month result comes from shelter costs of mortgage interest plus property tax alone, without the New York standard utility allowance. The SUA-inclusive excess shelter deduction is ~$611/month, which gives $194 of net income and $239.80/month (and $248.58 in October–December)." +us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and the mortgage-interest and property-tax shelter costs, but its $123/month benefit implies net income near $580 because it left out the New York standard utility allowance. With the SUA, the excess shelter deduction is ~$611, net income is $194, and the benefit is $239.80/month." +us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI correctly ($1,014 gross) but explicitly took no shelter deduction, ignoring the listed mortgage interest and property tax and the NY standard utility allowance. The uncapped elderly/disabled excess shelter deduction of ~$611/month cuts net income from $805 to $194." +us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reached the $298 maximum by treating net income as $0, which only happens if the $780.67/month of SSI is left out. With SSI counted, net income is $194 and the $58.20 contribution brings the benefit down to $239.80/month." +us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated SSI at $10,412/yr instead of $9,368 and used a $1,833/yr excess shelter deduction that leaves out the New York standard utility allowance; the correct deduction is ~$7,327/yr. That inflated annual net income to $8,871 instead of about $2,333." +us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and used the $209 deduction and $298 maximum, but it built shelter costs from mortgage interest and property tax only and left out the NY standard utility allowance. That produced $609 of net income instead of $194 and $115/month instead of $239.80." +us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without computing anything, although every input needed is listed: SS income, SSI eligibility from blindness/disability with low income, shelter costs, and assets. The formula gives $239.80/month for January–September and $248.58 for October–December." +us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It recognized SSI categorical eligibility but arrived at $735 of net income, which implies an excess shelter deduction far below the ~$611 that results once the NY standard utility allowance is added to mortgage interest and property tax. It also used a $301 maximum instead of $298." +us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It reached $0 net income by leaving the $780.67/month of SSI out of income, and it used the FY2025 $292 maximum. With SSI counted, net income is $194 and the benefit is $298 − $58.20 = $239.80/month." +us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI at $9,368 correctly but built the excess shelter deduction from mortgage interest and property tax alone and left out the New York standard utility allowance. That gave $7,305 of annual net income instead of about $2,333 and $1,384.50 instead of $2,903.94." +us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It treated shelter costs as $0 despite the listed mortgage interest and property tax, left out the $780.67/month of SSI, and used 2024 parameters ($193 standard deduction, $291 maximum). The correct computation counts $1,014 of gross income, subtracts $209 and a ~$611 excess shelter deduction, and gives $194 of net income." +us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only ~$233/month of Social Security and left out the $780.67/month of SSI, so net income hit $0 and it returned the $298×12 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month." +us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $780.67/month of SSI out of income, which drove net income to $0, and used an overstated $309 maximum allotment instead of $298. With SSI counted, net income is $194 and the benefit is $239.80/month." +us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated SSI at $867.67/month instead of $780.67 and took no shelter deduction, wrongly claiming no housing-cost input existed despite the listed mortgage interest and property tax. The uncapped excess shelter deduction, including the NY utility allowance, is ~$611/month, which leaves $194 of net income and a $239.80 benefit." +us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It said income was too low to qualify and that the standard deduction produces a $0 benefit, which inverts the SNAP formula: low net income raises the allotment. This SSI-categorically-eligible household receives $298 − $58.20 = $239.80/month." +us,scenario_118,snap,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and used the $209/$298 parameters, but its $196 excess shelter deduction comes from mortgage interest and property tax alone, without the New York standard utility allowance. The correct ~$611 deduction gives $194 of net income and $239.80/month (and $248.58 in October–December)." +us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It used $598.75 (mortgage interest plus property tax) as total shelter costs and left out the New York standard utility allowance, so its excess shelter deduction was $196.25 instead of ~$611. That left $608.75 of net income instead of $194 and $115.38/month instead of $239.80." +us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left out both SSI and the excess shelter deduction, subtracting only a $198 standard deduction from $233 of SS against a $291 maximum. The correct net income is $194 ($1,014 gross less $209 and ~$611), giving a $58.20 contribution and $239.80/month." +us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly assumed SNAP requires a qualifying child. Single elderly/disabled adults are eligible, and this SSI recipient passes all tests and receives $298 − $58.20 = $239.80/month." us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"The model's own reasoning correctly computed 12 × ($994 − $213.33) = $9,368. It then dropped that result, cited a made-up $725.33 FBR and an incoherent adjustment formula, and submitted $6,144. That number comes from no valid SSI computation." us,scenario_118,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model compared the $2,800 annual Social Security total against a monthly FBR and concluded that income exceeds the limit. In fact, countable income is only $213.33/month after the $20 exclusion, well below the $994 2026 FBR. It also used the stale 2024 FBR of $943." us,scenario_118,ssi,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model asserted that $2,800/year of Social Security exceeds the SSI income threshold. Monthly countable income is $233.33 − $20 = $213.33, far below the $994 FBR, so the federal payment is $780.67/month ($9,368/year)." @@ -9467,41 +9589,42 @@ us,scenario_118,ssi,kimi-k3,llm_error,state_local_rule,False,"The model correctl us,scenario_118,ssi,minimax-m3,llm_error,thresholds_rates,False,"The model stated that $233/month of Social Security exceeds a $967+ FBR, which is an inverted comparison because $233 is far below the FBR. Countable income of $213.33 against the $994 FBR leaves $780.67/month." us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"The model's reasoning computed a positive $8,756 benefit but it submitted 0 with no basis. That figure also relied on the stale 2024 FBR of $943 instead of the 2026 FBR of $994." us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model acknowledged that the head is blind and disabled but returned 0 without running the income test. Countable income of $213.33/month against the $994 FBR yields $780.67/month, or $9,368/year." -us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Wrongly said homeowners with these facts don't qualify for the circuit-breaker credit. The NY real property tax credit applies to homeowners who pay real property taxes and have household gross income of $18,000 or less. For a 65+ filer whose $1,634 in taxes far exceeds the income threshold, it pays the $375 elderly maximum even with zero NY tax." -us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly said NY has no general refundable credit that applies here and only looked at the Empire State EITC. It missed the refundable real property tax credit (IT-214), which gives this 74-year-old low-income homeowner the $375 elderly maximum because of the $1,634 in property taxes." -us,scenario_118,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Made refundable credits depend on having NY tax liability, earned income, or children. The real property tax credit depends only on property taxes paid, age, and household gross income of $18,000 or less, so this homeowner gets the $375 elderly maximum." -us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Correctly picked the senior real property tax credit and its $375 maximum, then gave an unexplained ~$63. The $1,634 in property taxes is far above the small income-based percentage of near-zero household income, so the cap binds and the credit is the full $375." -us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Wrongly said the real property tax credit needs tax liability to produce a refund. The credit is fully refundable and is paid with zero NY tax, so this 74-year-old homeowner gets $375." -us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated the lack of rent as ruling out the real property tax credit. Homeowners qualify through the real property taxes they pay, and $1,634 in taxes gives this 65+ filer the $375 maximum credit." -us,scenario_118,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Counted $2,800 of Social Security plus about $9,368 of estimated SSI as household gross income ($12,168). That puts the 65+ filer in the $12,001–$13,000 band with a $171 cap. The engine's household income measure for this credit leaves out these nontaxable benefits, so the filer falls in the lowest band and gets the full $375 elderly maximum." -us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly found the real property tax credit and that the computed credit is above the cap, but used the $75 maximum for filers under 65. The cap depends on whether a household member is 65 or older, not on dependents. For this 74-year-old the elderly maximum is $375." -us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Dismissed the real property tax credit as negligible and tied it to paying rent. A homeowner paying $1,634 in property taxes with almost no income qualifies, and the 65+ cap gives $375." -us,scenario_118,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Reported the NY household credit, which is nonrefundable and not part of state refundable credits. It also explicitly skipped the real property tax credit, which is the only refundable credit here and is worth $375." -us,scenario_118,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Assumed zero state tax liability rules out refundable credits. The NY real property tax credit is refundable regardless of liability and pays this 65+ homeowner $375. -us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Correctly got the $375 real property tax credit, then added a $75 NY household credit. The household credit is nonrefundable and only reduces tax liability, which is zero, so it adds nothing to refundable credits. The total stays at $375." -us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Said no NY refundable credit applies and never considered the real property tax credit. A 74-year-old homeowner with $1,634 in property taxes and household gross income under $18,000 gets the $375 elderly maximum." -us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Only checked the Empire State Child Credit and state EITC. It missed the refundable real property tax credit, which doesn't need earned income or dependents and pays this 65+ homeowner $375." -us,scenario_118,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Said no state refundable credit applies and overlooked the NY real property tax credit. The household's $1,634 in property taxes and the head's age of 74 qualify it for the $375 elderly maximum." -us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Overlooked the refundable NY real property tax credit for low-income homeowners. With $1,634 in property taxes and a 74-year-old head, the credit is the $375 elderly maximum." -us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Required earned income or dependents for any refundable credit. The NY real property tax credit needs neither and pays this 65+ homeowner $375 based on $1,634 in property taxes." -us,scenario_118,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Only ruled out the Empire State Child Credit and NY EITC, and never evaluated the real property tax credit. That credit pays this 74-year-old homeowner the $375 elderly maximum." -us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly found the real property tax credit and computed 50% of the taxes above 3.5% of income ($768), but applied a made-up $1,000 'expanded' maximum. The statutory cap for a 65+ household in the lowest income band is $375, so the credit is $375." -us,scenario_118,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Only checked the NY EITC and missed the refundable real property tax credit. A 74-year-old homeowner paying $1,634 in property taxes gets the $375 elderly maximum." -us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Said nothing triggers a NY refundable credit and missed the real property tax credit. The head's age (74), homeownership with $1,634 in property taxes, and household gross income under $18,000 qualify for the $375 elderly maximum." -us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Never evaluated the NY real property tax credit, the refundable circuit-breaker credit for low-income homeowners. With $1,634 in property taxes and a head aged 74, it pays $375." -us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Concluded that Social Security-only income produces no refundable credit, ignoring the real property tax credit. That credit depends on property taxes paid and age, not taxable income, and pays this homeowner the $375 elderly maximum." -us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Reported the NY household credit as refundable, but it is a nonrefundable credit that only offsets liability, which is zero here. It also missed the refundable real property tax credit of $375." -us,scenario_118,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Used a $75 cap for a homeowner aged 65 or older. $75 is the top of the under-65 maximum table, and the 65+ maximums run from $375 down to $86, so $75 is not an elderly cap at any income level. Here the elderly cap is $375." -us,scenario_118,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Made up a 2.5%-of-property-taxes formula. The credit is the property taxes above a small percentage of household gross income, capped at the 65+ maximum. With $1,634 in taxes and almost no income, that gives $375." -us,scenario_118,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Added Social Security and estimated SSI to household gross income, which put the 65+ filer in the $12,001–$13,000 band with a $171 cap. The engine's household income measure for this credit leaves out these nontaxable benefits, so the filer sits in the lowest band and gets the full $375 elderly maximum." -us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no NY credit applies and never evaluated the real property tax credit. That credit pays this 74-year-old homeowner with $1,634 in property taxes the $375 elderly maximum." -us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Treated the NY EITC and child credits as the only refundable credits. It missed the real property tax credit, which pays this low-income 65+ homeowner $375 regardless of earned income." -us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Wrongly said property tax credits don't apply here. A 74-year-old homeowner paying $1,634 in property taxes with household gross income under $18,000 qualifies for the refundable real property tax credit at the $375 elderly maximum." -us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Went through EITC, child, tuition, and child care credits but left out the real property tax credit. The household qualifies for that credit through its $1,634 in property taxes and the head's age of 74, worth $375." -us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,Made refundable credits depend on NY tax liability or children. The real property tax credit is paid with zero liability and gives this 65+ homeowner $375. -us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Only checked the NY EITC and the refundable child and dependent care credit. It missed the real property tax credit, which needs no earnings or dependents and pays this 74-year-old homeowner $375." -us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Tied refundable credits to taxable income, earned income, or children. The NY real property tax credit depends on property taxes paid, age 65+, and household gross income of $18,000 or less, and pays $375 here." -us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Wrongly said age and income rule out every NY refundable credit. The head's age of 74 is exactly what makes them eligible for the elderly real property tax credit, which pays the $375 maximum against $1,634 in property taxes." +us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model asserted that a homeowner with these facts does not qualify for the NY real property tax circuit breaker. In fact this 74-year-old homeowner has federal AGI of $0 and $1,634 of property tax, above 3.5% of that AGI, so the flat table gives $375." +us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model looked only at earned-income-based credits and concluded there was no refundable NY credit. It never applied the IT-214 real property tax credit: this homeowner is 74 with federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375." +us,scenario_118,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model tied NY refundable credits to having tax liability, earned income, or children. The real property tax credit requires none of these and pays this elderly homeowner the $375 cap." +us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model identified the senior RPTC and mentioned $375, then substituted an unexplained $63. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives a flat $375, so nothing reduces it to $63." +us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model assumed the real property tax credit needs NY taxable income or liability to generate a refund. The credit is fully refundable regardless of liability and pays $375 to this senior homeowner with $2,800 HGI." +us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model dismissed the real property tax credit because no rent was reported. Homeowners qualify through real estate taxes paid, and the $1,634 here yields the $375 elderly maximum." +us,scenario_118,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model used the repealed household-gross-income rule, including estimated SSI, to select a $171 income-band cap. Under the rule in force from 2025, this 74-year-old filer has federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375." +us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula with a 6% threshold and applied a $75 maximum it tied to having no dependents. The head is 74 with federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375." +us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model rejected the real property tax credit because no rent was paid and called the senior credit negligible. Homeowners can qualify through property taxes; with federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old filer receives a flat $375." +us,scenario_118,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model reported the NY household credit, which is nonrefundable, as the refundable amount, and said it explicitly skipped the real property tax credit. The refundable RPTC for this senior homeowner is $375." +us,scenario_118,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,The model treated zero state tax liability as ruling out refundable credits. The NY real property tax credit is paid regardless of liability and gives this 74-year-old homeowner $375. +us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly reached the $375 RPTC senior cap but added a $75 NY household credit. That credit is nonrefundable, so it doesn't belong in state refundable credits, and the correct total is $375 rather than $450." +us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model concluded no NY refundable credit applies and overlooked the real property tax credit. With $2,800 HGI and $1,634 in property taxes, this senior homeowner receives the $375 elderly maximum." +us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model checked only the Empire State Child Credit and NY EITC. It never evaluated the IT-214 real property tax credit, which pays this 74-year-old homeowner the $375 cap." +us,scenario_118,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model asserted no refundable credits apply and omitted the NY real property tax credit. This 74-year-old homeowner has federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, so the flat table gives $375." +us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model missed the refundable NY real property tax credit. With federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old homeowner receives a flat $375." +us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model required earned income or dependents for any refundable credit. The NY real property tax credit applies here because this 74-year-old filer has federal AGI of $0 and property taxes above 3.5% of that AGI, giving a flat $375." +us,scenario_118,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,The model ruled out the child credit and EITC but never considered the real property tax credit. That credit gives this 74-year-old homeowner a flat $375 because federal AGI is $0 and property taxes exceed 3.5% of that AGI. +us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula to calculate $768 and compared it with a $1,000 senior maximum that does not apply here. With federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old filer receives a flat $375." +us,scenario_118,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model considered only the NY EITC and concluded nothing applies without earned income. It missed the refundable real property tax credit, which pays this 74-year-old homeowner $375." +us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said no refundable NY credit was triggered and overlooked that the listed age, real estate taxes, and low Social Security-only income qualify the head for the real property tax credit at its $375 elderly cap." +us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model found no refundable credit and missed the IT-214 real property tax credit. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives a flat $375." +us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model concluded that Social Security-only income generates no refundable credit. The real property tax credit uses federal AGI and a flat table; with federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old homeowner receives $375." +us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model reported the $90 NY household credit, which is nonrefundable, as refundable, and it omitted the real property tax credit. The refundable amount is the $375 RPTC elderly maximum." +us,scenario_118,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model used the repealed household-gross-income rule, including estimated SSI, and applied a $75 cap it described as the cap for a homeowner aged 65 or older. The head is 74 with federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375." +us,scenario_118,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model computed the credit as 2.5% of property taxes, which is not the IT-214 formula. From 2025 the credit uses federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives $375." +us,scenario_118,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model used the repealed household-gross-income rule, including estimated SSI, to select a $171 income-band cap. With federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old filer receives a flat $375." +us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model found no qualifying credits and missed the NY real property tax credit. A 74-year-old homeowner with $2,800 HGI and $1,634 in property taxes receives the $375 maximum." +us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model treated the EITC as NY's main refundable credit and never evaluated the real property tax credit, which pays this senior homeowner $375 regardless of earned income." +us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model explicitly dismissed property tax credits. The NY real property tax credit applies to this 74-year-old homeowner with federal AGI of $0 and property taxes above 3.5% of that AGI, giving a flat $375." +us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model checked only the EITC, child, childcare, and tuition credits. It missed the refundable real property tax credit, which the listed real estate taxes and age 74 trigger at the $375 cap." +us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model required NY tax liability or qualifying children. The real property tax credit is refundable without liability and gives this elderly homeowner $375. +us,scenario_118,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"The model concluded there was no basis for a refundable NY credit and overlooked that $1,634 in real estate taxes and age 74 qualify the head for the real property tax credit at its $375 elderly maximum." +us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model considered only the NY EITC and the refundable CDCC. It never evaluated the real property tax credit, which gives this 74-year-old homeowner a flat $375 based on federal AGI of $0 and property taxes above 3.5% of that AGI." +us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model equated NY refundable credits with the earned-income and child credits and missed the real property tax credit. The listed age and property taxes produce that credit at $375. +us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model claimed the household's income and age qualify it for no credit. Age 74, federal AGI of $0, and property taxes above 3.5% of that AGI qualify it for the NY real property tax credit at a flat $375." us,scenario_119,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Tested only age under 19 and income of about 197% FPL against the 200% FAMIS limit. It ignored that Child 1 has employer-sponsored insurance, and a child covered under a group health plan is not a CHIP targeted low-income child." us,scenario_119,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"Correctly placed income (about 197% FPL) under the 200% FAMIS limit plus the 5% disregard. It then stopped there and never applied CHIP's rule excluding children already covered by employer-sponsored insurance, which Child 1 has." us,scenario_119,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"Checked MAGI of $52,569 (about 197% FPL) against the 200% CHIP limit and confirmed the child is not Medicaid-eligible. It skipped the remaining categorical condition: Child 1 is covered by employer-sponsored insurance, which bars CHIP eligibility." @@ -9577,53 +9700,54 @@ us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error, us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $12,250 post-TCJA head-of-household deduction, three $5,300 personal exemptions, 15% brackets, and a $2,000 total CTC. OBBBA keeps the $24,150 deduction with no exemptions and a $2,200-per-child CTC. That CTC absorbs the $2,310 left after the $546 CDCC and $200 Saver's credit." us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It gave no parseable value and no explanation for federal_income_tax_before_refundable_credits. us,scenario_119,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It filed as single with a $16,100 standard deduction and single brackets, which gave $5,328 of tax. A parent with two qualifying children files as head of household with a $24,150 deduction, and that tax is $3,056.23. It also used a 20% CDCC instead of 35% and left out the Saver's credit. Under head-of-household status, its correct $4,400 CTC fully offsets the liability." -us,scenario_119,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It set the EITC to $0, but the 2026 two-child EITC phases out at 21.06% above $23,890 and is still $764.23 at $55,000 of earnings. It then set the ACTC at the $3,400 cap ($1,700 × 2) instead of the unused CTC. That is $4,400 minus the $2,310.23 net liability left after the $546 CDCC and $200 Saver's Credit, or $2,089.77." -us,scenario_119,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It computed the ACTC as $4,400 − $2,514 and left out the $200 Saver's Credit (10% of $2,000 of retirement contributions at AGI $52,569). That credit lowers the liability the CTC absorbs to $2,310.23, so the ACTC is $2,089.77. Its $1,268 EITC is phased out from AGI $52,569 rather than PolicyEngine's $55,000 earned-income base (the greater of AGI and earned income), which yields $764.23." -us,scenario_119,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It wrongly said $55,000 is past the EITC phase-out and that the Additional CTC disappears at this income. The two-child EITC is still $764.23 at $55,000. The ACTC has no income ceiling at this level and refunds the $2,089.77 of the $4,400 CTC that exceeds the $2,310.23 net tax liability." -us,scenario_119,federal_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"It said there was no EITC and no CTC left over. Net tax before the CTC is only $2,310.23 after the $546 CDCC and $200 Saver's Credit, so $2,089.77 of the $4,400 CTC is refundable. The two-child EITC is still $764.23 at $55,000 of earnings." -us,scenario_119,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used the pre-OBBBA $2,000-per-child CTC ($4,000) and a ~$3,100 tax that ignores the $546 CDCC and $200 Saver's Credit. That gave a $900 ACTC instead of $4,400 − $2,310.23 = $2,089.77. It also zeroed the EITC by putting the two-child phase-out end near $52,900; the 2026 credit runs out near $58,629 and is $764.23 at $55,000." -us,scenario_119,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It claimed ~$48,300 of taxable income and ~$5,000 of tax after a $32,300 deduction, which overstates liability. Under HoH rules, $52,569 − $24,150 = $28,419 of taxable income and $3,056.28 of tax, which the CDCC and Saver's Credit cut to $2,310.23. That leaves $2,089.77 of the $4,400 CTC refundable; the model also used $2,000 per child. It also zeroed the $764.23 EITC." -us,scenario_119,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It zeroed the EITC, which is $764.23 at $55,000 of earnings with two children. It also capped the ACTC at a $1,800-per-child figure and assumed tax absorbed only ~$800 of the CTC. The CTC actually offsets a $2,310.23 net liability, so the ACTC is $4,400 − $2,310.23 = $2,089.77." -us,scenario_119,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It left out the $200 Saver's Credit (10% of $2,000 of qualifying retirement contributions at AGI $52,569). Its pre-CTC liability of $2,510.28 was therefore $200 too high, giving an ACTC of $1,889.72 instead of $2,089.77. Its $1,276.20 EITC phases out from AGI $52,569, while PolicyEngine's phase-out base is the $55,000 of wages, which yields $764.23." -us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used the pre-OBBBA $2,000-per-child CTC and a $22,500 HoH standard deduction, and ignored both the $546 CDCC and the $200 Saver's Credit. That gave a $723 ACTC instead of $2,089.77. It also put the EITC at $922 using guessed parameters on $51,913 of earnings, versus $764.23 on the $55,000 base, then rounded $1,645 down to $1,600." -us,scenario_119,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It assumed tax liability fully absorbs the CTC and that income is too high for the EITC. Tax before the CTC is only $2,310.23 after the CDCC and Saver's Credit, so $2,089.77 of the $4,400 CTC is refundable. The two-child EITC is $764.23 at $55,000." -us,scenario_119,federal_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It applied a 20% CDCC rate ($312) instead of the 35% rate OBBBA sets for 2026 at this AGI ($546), and left out the $200 Saver's Credit. Its ACTC was therefore $1,642 rather than $2,089.77. Its $1,276 EITC uses AGI $52,569 as the phase-out base instead of the $55,000 of earnings, which gives $764.23." -us,scenario_119,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It cut income to $46,180 by subtracting the $6,389 employer-sponsored insurance premiums, though PolicyEngine's AGI is $52,569 and the EITC phase-out base is $55,000. With wrong parameters, that inflated the EITC to $3,913 versus $764.23. It also used a $2,000-per-child CTC and a $2,403 liability, giving an ACTC of $1,597 instead of $2,089.77." -us,scenario_119,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It said the nonrefundable CTC was fully used, but the $4,400 2026 CTC is set against only $2,310.23 of net tax, leaving a $2,089.77 refundable ACTC. Its $1,431 EITC uses misremembered parameters (max $7,264, start $24,871) on AGI. The 2026 values are $7,316 and $23,890, and on the $55,000 earnings base they give $764.23." -us,scenario_119,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It set the ACTC to $0, claiming the CTC was fully used nonrefundably. A $4,400 CTC against $2,310.23 of net tax leaves $2,089.77 refundable. Its $1,075.76 EITC also exceeds the $764.23 from phasing out at 21.06% above $23,890 on $55,000 of earnings." -us,scenario_119,federal_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It treated a $2,000 CTC as fully absorbed and set the ACTC to $0. The 2026 credit is $2,200 per child ($4,400), which exceeds the $2,310.23 net liability by $2,089.77. Its EITC used wrong parameters (max $7,350, start $22,800) on AGI $52,569, giving $1,080.65 instead of $764.23." -us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 employer insurance premiums, which inflated the EITC to $2,643 versus $764.23 on the $55,000 earnings base. It also used a $2,000 per-child CTC and refunded only $427. The correct ACTC is $4,400 minus $2,310.23 of net tax, or $2,089.77." -us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It stated a flat $3,000 refundable CTC without computing tax liability, and left out the EITC entirely. The unused CTC is $4,400 − $2,310.23 = $2,089.77, and the EITC adds $764.23." -us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It refunded only $172 of CTC, so it treated far more of the credit as absorbed than the $2,310.23 net liability. That liability comes from the $24,150 HoH deduction, the $546 CDCC and the $200 Saver's Credit, and it leaves a $2,089.77 ACTC. Its $1,933 EITC is too high; the 21.06% phase-out on $55,000 gives $764.23." -us,scenario_119,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It subtracted both the $3,087 401(k) deferral and the $6,389 employer insurance premiums to get $45,524 of earned income. That put the EITC at $1,808 versus $764.23 on the $55,000 phase-out base. It also set the ACTC to $0, though $2,089.77 of the $4,400 CTC exceeds the $2,310.23 net liability." -us,scenario_119,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It returned $0 with no computation. That misses both the $2,089.77 ACTC (the CTC left over after a $2,310.23 net liability) and the $764.23 EITC still available to a two-child HoH filer at $55,000 of earnings." -us,scenario_119,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It gave $2,216.39 with no breakdown, $637.62 short of the correct total. That total is the $764.23 EITC plus a $2,089.77 ACTC, where the ACTC is the $4,400 CTC minus $2,310.23 of net liability after the $546 CDCC and $200 Saver's Credit." -us,scenario_119,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"Its $1,530.82 is below the $2,089.77 ACTC alone, so it understated the unused CTC. That is $4,400 minus a net liability of only $2,310.23 after the $546 CDCC and $200 Saver's Credit, and the $764.23 EITC comes on top of it." -us,scenario_119,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"Its $1,044 ACTC implies a pre-CTC liability well above the actual $2,310.23 (tax of $3,056.28 less the $546 CDCC and $200 Saver's Credit); the correct ACTC is $2,089.77. Its $497 EITC is below the $764.23 that results from phasing out $7,316 at 21.06% above $23,890 on $55,000." -us,scenario_119,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It used a $4,000 CTC ($2,000 per child) instead of the 2026 $2,200-per-child $4,400, and overstated nonrefundable use at $2,894 versus $2,310.23. That gave a $1,106 ACTC instead of $2,089.77. It also zeroed the EITC, claiming $51,913 is past the phase-out, but the two-child 2026 phase-out ends near $58,629 and the EITC is $764.23." -us,scenario_119,federal_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no answer for federal_refundable_credits. The correct value is the $764.23 EITC plus the $2,089.77 ACTC, or $2,854.01." -us,scenario_119,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It treated the full $2,000 per child as refundable. That ignores that the CTC first offsets the $2,310.23 net liability, that the 2026 credit is $2,200 per child, and that the refundable part is capped at $1,700 per child. The ACTC is $2,089.77, and the model also left out the $764.23 EITC." -us,scenario_119,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It gave $190 with no derivation, missing nearly all of the $2,089.77 ACTC ($4,400 CTC minus $2,310.23 of net tax) and the $764.23 EITC." -us,scenario_119,federal_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It said the CTC was fully used nonrefundably, but net tax after the $546 CDCC and $200 Saver's Credit is only $2,310.23 against a $4,400 CTC, so $2,089.77 is refundable. Its $1,496 EITC also exceeds the $764.23 obtained on the $55,000 phase-out base." -us,scenario_119,federal_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"It put the EITC at about $3,000 instead of $764.23 (the $7,316 maximum less 21.06% of $55,000 − $23,890). It also overstated the ACTC at $2,363.10 versus $2,089.77 by understating the $2,310.23 net liability the CTC offsets." -us,scenario_119,federal_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"Its $1,656 ACTC reflects a 20% CDCC ($312) instead of the 2026 35% rate ($546) and no $200 Saver's Credit; the correct ACTC is $2,089.77. Its $1,276 EITC phases out from AGI $52,569 rather than the $55,000 earnings base, which gives $764.23." -us,scenario_119,federal_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"Its $1,656 ACTC uses a $312 CDCC (20%) instead of $546 (35%) and leaves out the $200 Saver's Credit. That understates the ACTC by about $434 against $2,089.77, while its $785 EITC is close to $764.23." -us,scenario_119,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It subtracted the $6,389 employer insurance premiums to reach an AGI of $46,180. That understated tax, so only $1,543.60 of CTC was used instead of $2,310.23 and the ACTC rose to $2,856.40 instead of $2,089.77. It also produced a $2,621.73 EITC instead of $764.23 on the $55,000 phase-out base." -us,scenario_119,federal_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"Its $1,344 ACTC is $4,400 minus the full $3,056 regular tax. It left out the $546 CDCC and $200 Saver's Credit, which cut liability to $2,310.23 and raise the ACTC to $2,089.77. Its $1,246 EITC also exceeds the $764.23 from the 21.06% phase-out on $55,000." -us,scenario_119,federal_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"Its ACTC of $1,343.72 = $4,400 − $3,056.28 ignores the $546 CDCC and $200 Saver's Credit applied before the CTC, so the ACTC should be $2,089.77. Its ~$1,474 EITC is above the $764.23 from the 21.06% phase-out on $55,000." -us,scenario_119,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It applied the $546 CDCC and $200 Saver's Credit, but subtracted the $6,389 employer insurance premiums to reach an AGI of $46,180. That understated tax at $2,289.60, so the ACTC became $2,856.40 instead of $2,089.77. The EITC came out $2,623.83 instead of $764.23 on the $55,000 phase-out base." -us,scenario_119,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It said income is past the EITC phase-out and missed the ACTC entirely. At $55,000 the two-child EITC is still $764.23, and the $4,400 CTC exceeds the $2,310.23 net liability by $2,089.77, all of it refundable." -us,scenario_119,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It used a $2,000 total CTC, entirely absorbed as nonrefundable. The 2026 CTC is $2,200 per child ($4,400) against $2,310.23 of net tax, leaving a $2,089.77 ACTC. Its $1,300 EITC phases out from AGI $52,569 instead of the $55,000 earnings base, which gives $764.23." -us,scenario_119,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used a $1,000-per-child CTC instead of $2,200 and so found no ACTC, whereas $4,400 minus $2,310.23 of net liability leaves $2,089.77 refundable. Its $1,115 EITC uses AGI $52,569 and a $23,200 start, versus the $23,890 start on the $55,000 earnings base, which gives $764.23." -us,scenario_119,federal_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"Its $806 ACTC implies a pre-CTC liability far above the $2,310.23 net tax (regular tax of $3,056.28 less the $546 CDCC and $200 Saver's Credit); the ACTC is $2,089.77. Its $1,301 EITC phases out from AGI $52,569 instead of $55,000, which gives $764.23." -us,scenario_119,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"Its $4,109 EITC skips most of the 21.06% phase-out from $23,890 to $55,000, which leaves $764.23. Its $584 ACTC overstates how much CTC tax absorbs: only $2,310.23, leaving $2,089.77 refundable." -us,scenario_119,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"Its errors nearly cancelled. The $1,658 ACTC used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's Credit, understating the ACTC by about $432 against $2,089.77. Its $1,188 EITC phases out on $51,913 rather than the $55,000 earnings base, which yields $764.23." -us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer for federal_refundable_credits. The correct value is the $764.23 EITC plus the $2,089.77 ACTC, or $2,854.01." -us,scenario_119,federal_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It computed tax as a single filer ($16,100 deduction and a $12,400 10% bracket, giving $4,128.28) instead of head of household ($24,150 and $17,700, giving $3,056.28). It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's Credit. That made the ACTC $583.72 instead of $2,089.77; its $764.23 EITC was right." -us,scenario_119,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It said $55,000 exceeds the EITC limits and that tax exceeds the CTC. The two-child EITC phases out completely only near $58,629 and is $764.23 at $55,000. Net tax of $2,310.23 is well below the $4,400 CTC, leaving $2,089.77 refundable." -us,scenario_119,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It put the two-child EITC ceiling near $48,000, but the 2026 phase-out ends near $58,629, which gives $764.23 at $55,000. It also said the entire $4,400 CTC was absorbed, though net tax after the CDCC and Saver's Credit is $2,310.23, leaving a $2,089.77 ACTC." -us,scenario_119,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It treated the single parent as married filing jointly and used an invented ~$49,084 phase-out start on $55,800 of income. That produced a $6,400 EITC instead of $764.23 under head-of-household rules. It also used a $4,000 CTC, giving an ACTC of $1,837 instead of $2,089.77." -us,scenario_119,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It denied the EITC because of the $300,000 in stock, but the EITC has no asset test, only an investment-income limit, and $800 of interest is far below it. The EITC is $764.23. Its $565 ACTC also understates the $2,089.77 left after the $4,400 CTC offsets $2,310.23 of net tax." +us,scenario_119,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It set the EITC to zero, saying AGI of $52,569 is above the two-child phase-out. In fact the 2026 head-of-household phase-out runs from $23,890 to about $58,630 and leaves $1,276.30. It also assumed the $1,700-per-child cap limits the ACTC to $3,400. The refundable CTC is actually the $2,089.77 left over after $2,310.23 of liability (tax of $3,056.23 minus the $546 CDCC and the $200 saver's credit)." +us,scenario_119,federal_refundable_credits,claude-fable-5.1,llm_error,other,False,"Its $2,514 of nonrefundable CTC use is the $3,056 tax minus only the $546 CDCC. It left out the $200 saver's credit (10% of $2,000 of retirement contributions), which brings liability down to $2,310.23 and raises the refundable CTC to $2,089.77. It also started the EITC phase-out at about $23,850 instead of $23,890, giving $1,268 instead of $1,276.30." +us,scenario_119,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It treated $55,000 as above the EITC phase-out and assumed the refundable CTC is unavailable at this income. The 2026 head-of-household two-child EITC stays positive up to about $58,630, so AGI of $52,568.54 gives $1,276.30. The ACTC has no income cutoff here, so the $2,089.77 of CTC left after $2,310.23 of liability is refundable." +us,scenario_119,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It said there is no EITC and no CTC left over. The two-child EITC phases out at 21.06% above $23,890, which leaves $1,276.30 at AGI of $52,568.54. The $4,400 CTC is larger than the $2,310.23 of liability left after the CDCC and saver's credit, so $2,089.77 is refundable." +us,scenario_119,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It used the old $2,000-per-child CTC, skipped the $546 CDCC and $200 saver's credit that reduce liability before the CTC, and thought the two-child EITC ran out near $52,900. In 2026 the CTC is $2,200 per child and liability after the other credits is $2,310.23, so $2,089.77 is refundable. The EITC is $1,276.30 because the phase-out runs from $23,890 to about $58,630." +us,scenario_119,federal_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used a $32,300 married-filing-jointly deduction inconsistently, estimated tax above $5,000, and concluded that the $2,000-per-child CTC is fully used and the EITC is zero. As head of household with a $24,150 deduction, tax is $3,056.23, which falls to $2,310.23 after the CDCC and saver's credit, so $2,089.77 of the $4,400 CTC is refundable. The EITC at AGI of $52,568.54 is $1,276.30." +us,scenario_119,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It dropped the EITC as 'far above' the limit, but the two-child phase-out leaves $1,276.30 at AGI of $52,568.54. It also assumed only about $800 of CTC is used against tax and applied an $1,800-per-child cap. Liability after the CDCC and saver's credit is actually $2,310.23, so the refundable CTC is $2,089.77, under the $1,700-per-child cap." +us,scenario_119,federal_refundable_credits,claude-opus-5.5,llm_error,other,False,"It figured $2,510.28 of nonrefundable CTC use by subtracting only the $546 CDCC from tax. It left out the $200 saver's credit (10% rate on $2,000 of contributions at head-of-household AGI of $52,569), which lowers liability to $2,310.23 and raises the refundable CTC to $2,089.77. Its $1,276.20 EITC is right." +us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the old $2,000-per-child CTC and a $22,500 head-of-household deduction, never subtracted the $546 CDCC or $200 saver's credit, and phased out the EITC on earned income with guessed parameters ($6,780 maximum, $24,100 start). It then rounded to $1,600. The 2026 values are a $2,200-per-child CTC, a $24,150 deduction, and a $7,316 maximum EITC phasing out from $23,890 on AGI of $52,568.54. That gives a $1,276.30 EITC plus a $2,089.77 ACTC." +us,scenario_119,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It assumed tax fully uses up a $4,000 CTC and that the EITC is phased out. With the $24,150 head-of-household deduction, tax is $3,056.23, cut to $2,310.23 by the CDCC and saver's credit, so $2,089.77 of the $4,400 CTC is refundable. The EITC is $1,276.30 because the two-child phase-out extends to about $58,630." +us,scenario_119,federal_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used the old 20% CDCC rate ($312) instead of the 2026 rate of 35% ($546) at AGI of $52,569, and it left out the $200 saver's credit. That left $2,758 of liability for the CTC instead of $2,310.23, so its refundable CTC was $1,642 instead of $2,089.77." +us,scenario_119,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It based the EITC on AGI of $46,180, which means it subtracted the $6,389 employer health premium from the stated gross wages. AGI is $52,568.54, which gives an EITC of $1,276.30, not about $3,913. Its refundable CTC also used a $4,000 CTC against a $2,403 tax instead of $4,400 minus $2,310.23 = $2,089.77." +us,scenario_119,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It concluded the CTC is fully used against tax, but the 2026 CTC is $4,400 and liability after the $546 CDCC and $200 saver's credit is only $2,310.23, so $2,089.77 is refundable. Its EITC also used a $24,871 phase-out start and a $7,264 maximum instead of $23,890 and $7,316." +us,scenario_119,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"It set the refundable CTC to zero even though the $4,400 CTC is larger than the $2,310.23 of liability left after the CDCC and saver's credit, so it left out $2,089.77. Its $1,075.76 EITC is also too low: the $7,316 maximum phased out at 21.06% above $23,890 gives $1,276.30." +us,scenario_119,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It treated the CTC as a single $2,000 credit, fully used against tax. Two children get $2,200 each ($4,400), and only $2,310.23 of liability remains, so $2,089.77 is refundable. Its EITC also used a $22,800 phase-out start and a $7,350 maximum instead of $23,890 and $7,316." +us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used AGI of $46,180 by subtracting the $6,389 employer health premium from the stated gross wages, which pushed the EITC up to $2,643. It also treated the CTC as one $2,000 credit and left $427 refundable. AGI is $52,568.54 (EITC of $1,276.30), and the CTC is $4,400, with $2,089.77 refundable after $2,310.23 of liability." +us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It reported $3,000 of refundable CTC with no EITC and no liability calculation. It left out the EITC entirely, though AGI of $52,568.54 is still inside the two-child phase-out range and gives $1,276.30. The refundable CTC is the $4,400 CTC minus $2,310.23 of liability after the CDCC and saver's credit, which is $2,089.77." +us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Its $172 ACTC means it treated nearly the whole CTC as used against tax. The $546 CDCC and $200 saver's credit cut liability to $2,310.23, leaving $2,089.77 refundable. Its $1,933 EITC is too high: phasing the $7,316 maximum down at 21.06% on AGI of $52,568.54 above $23,890 gives $1,276.30." +us,scenario_119,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It figured earned income as $45,524 by subtracting the $6,389 employer health premium from gross wages, and it said the CTC was fully used against tax. Earned income is $51,912.80, and the phase-out uses AGI of $52,568.54, giving an EITC of $1,276.30. The $4,400 CTC exceeds the $2,310.23 of remaining liability, so $2,089.77 is refundable." +us,scenario_119,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It returned $0 with no working. It missed the $1,276.30 EITC (AGI of $52,568.54 is inside the two-child phase-out, which ends near $58,630) and the $2,089.77 refundable CTC left after $2,310.23 of liability uses part of the $4,400 CTC." +us,scenario_119,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $2,216.39 matches a $1,276.30 EITC plus about $940 of ACTC, which is what you get from the old $4,000 CTC set against the full $3,056 tax with no CDCC or saver's credit. The 2026 CTC is $4,400 and liability after the $546 CDCC and $200 saver's credit is $2,310.23, so the ACTC is $2,089.77." +us,scenario_119,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It gave no breakdown. Its $1,530.82 is $1,835.25 short, which fits a roughly correct $1,276 EITC plus only about $255 of ACTC, meaning it treated almost the whole CTC as used against tax. Liability after the $546 CDCC and $200 saver's credit is only $2,310.23, so $2,089.77 of the $4,400 CTC is refundable." +us,scenario_119,federal_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Its EITC of about $497 and ACTC of about $1,044 are both too low. The two-child EITC is $7,316 minus 21.06% of AGI above $23,890, which is $1,276.30. The ACTC is $4,400 minus $2,310.23 of liability (after the $546 CDCC and $200 saver's credit), which is $2,089.77. It used outdated EITC parameters and overstated how much CTC is used against tax." +us,scenario_119,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It set the EITC to zero, saying $51,913 is above the 2026 phase-out. The head-of-household two-child EITC actually runs out near $58,630 and equals $1,276.30 at AGI of $52,568.54. It also used a $4,000 CTC instead of $4,400 and $2,894 of nonrefundable use instead of $2,310.23, so its ACTC was $1,106 instead of $2,089.77." +us,scenario_119,federal_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_refundable_credits, so there is no answer to score." +us,scenario_119,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It treated the full $2,000 per child as refundable and left out the EITC. The refundable ACTC is only the CTC not used against the $2,310.23 liability ($2,089.77, within the $1,700-per-child cap), and the $1,276.30 EITC has to be added." +us,scenario_119,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It gave $190 with no working, missing almost all of the $1,276.30 EITC and the $2,089.77 refundable CTC. AGI of $52,568.54 is inside the two-child EITC phase-out, and the $4,400 CTC exceeds the $2,310.23 of remaining liability." +us,scenario_119,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It said the CTC is fully used against tax, but the 2026 CTC of $4,400 exceeds the $2,310.23 liability left after the $546 CDCC and $200 saver's credit, so $2,089.77 is refundable. Its $1,496 EITC is also too high: the $7,316 maximum phased out above $23,890 gives $1,276.30." +us,scenario_119,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Its EITC of about $3,000 is too high: the $7,316 maximum minus 21.06% of the $28,678.54 by which AGI exceeds $23,890 is $1,276.30. Its $2,363.10 refundable CTC is also too high because it understated liability, which is $2,310.23 after the CDCC and saver's credit, leaving $2,089.77." +us,scenario_119,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $1,656 ACTC equals $4,400 minus the $3,056 tax less a $312 CDCC. That means it used the old 20% CDCC rate instead of the 2026 rate of 35% ($546) and left out the $200 saver's credit. Liability left for the CTC is actually $2,310.23, so the ACTC is $2,089.77." +us,scenario_119,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $1,656 ACTC reflects a 20% ($312) CDCC and no $200 saver's credit, instead of a 35% ($546) CDCC plus the saver's credit, which leave $2,089.77 refundable. Its $785 EITC is also too low: phasing out on AGI of $52,568.54 from $23,890 gives $1,276.30." +us,scenario_119,federal_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its EITC uses AGI of $46,180, which means it subtracted the $6,389 employer health premium from the stated gross wages. With AGI of $52,568.54 the EITC is $1,276.30, not $2,621.73. The higher AGI also raises tax, so only $2,089.77 of the CTC is refundable, not $2,856.40." +us,scenario_119,federal_refundable_credits,gpt-6-luna,llm_error,other,False,"Its $1,344 ACTC equals $4,400 minus the full $3,056 tax, so it never subtracted the $546 CDCC and $200 saver's credit, which apply before the CTC. Liability left for the CTC is $2,310.23, so the ACTC is $2,089.77. Its $1,246 EITC is also $30 below $1,276.30." +us,scenario_119,federal_refundable_credits,gpt-6-sol,llm_error,other,False,"Its $1,343.72 ACTC is $4,400 minus the full $3,056.28 tax. It ignored the $546 CDCC and $200 saver's credit that reduce liability first, which understates the ACTC by $746. Its EITC of about $1,474 is too high: phasing out on AGI of $52,568.54 above $23,890 gives $1,276.30." +us,scenario_119,federal_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $2,623.83 EITC reflects AGI near $46,180, meaning it subtracted the $6,389 employer health premium from the stated gross wages. AGI is $52,568.54, which gives an EITC of $1,276.30. The higher AGI also means higher tax, leaving only $2,089.77 of the CTC refundable instead of $2,856.40." +us,scenario_119,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It said income is above the EITC phase-out and no other refundable credit applies. The two-child EITC is $1,276.30 at AGI of $52,568.54, and $2,089.77 of the $4,400 CTC is left after $2,310.23 of liability and is refundable." +us,scenario_119,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It treated the CTC as a single $2,000 credit fully used against tax. The 2026 CTC is $2,200 per child ($4,400), set against only $2,310.23 of liability after the CDCC and saver's credit, so $2,089.77 is refundable. Its $1,300 EITC is close to $1,276.30." +us,scenario_119,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the pre-2018 CTC of $1,000 per child and so found no ACTC. The 2026 CTC is $2,200 per child, and the $2,089.77 that exceeds the $2,310.23 liability is refundable. Its EITC also used a $23,200 start and a $7,300 maximum instead of $23,890 and $7,316, giving $1,115 instead of $1,276.30." +us,scenario_119,federal_refundable_credits,grok-4.7,llm_error,other,False,"Its $806 ACTC overstates the CTC used against tax by about $1,284. Liability after the $546 CDCC and $200 saver's credit is only $2,310.23, so $2,089.77 of the $4,400 CTC is refundable. Its $1,301 EITC is close to $1,276.30." +us,scenario_119,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Its EITC of $4,109 is far too high: the $7,316 maximum minus 21.06% of AGI above $23,890 is $1,276.30 at AGI of $52,568.54. Its $584 ACTC is too low: $4,400 of CTC minus $2,310.23 of liability leaves $2,089.77 refundable." +us,scenario_119,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its $2,742 of nonrefundable CTC use reflects a 20% ($312) CDCC and no $200 saver's credit. With the 2026 CDCC rate of 35% ($546) and the saver's credit, liability is $2,310.23 and the ACTC is $2,089.77. Its $1,188 EITC is also below $1,276.30." +us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_refundable_credits, so there is no answer to score." +us,scenario_119,federal_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It computed $4,128 of tax using the single-filer deduction and brackets instead of head of household ($24,150 deduction, $3,056.23 tax). It also used a 20% ($312) CDCC instead of 35% ($546) and left out the $200 saver's credit, so its ACTC was $583.72 instead of $2,089.77. Its $764.23 EITC phases out on $55,000 of gross wages instead of AGI of $52,568.54, which gives $1,276.30." +us,scenario_119,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It said income is too high for the EITC and that tax exceeds the CTC. The two-child EITC is $1,276.30 at AGI of $52,568.54. Head-of-household tax of $3,056.23, less the CDCC and saver's credit, leaves $2,310.23, which is below the $4,400 CTC, so $2,089.77 is refundable." +us,scenario_119,federal_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"Both parts are too low. Its $1,504 ACTC implies about $2,896 of liability used the CTC, but after the $546 CDCC and $200 saver's credit only $2,310.23 remains, leaving $2,089.77. Its $1,096 EITC falls short of $1,276.30." +us,scenario_119,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It put the two-child EITC cutoff near $48,000 and said the whole $4,400 CTC was used against tax. The 2026 head-of-household phase-out runs at 21.06% from $23,890 and ends near $58,630 (EITC of $1,276.30). Tax before credits is only $3,056.23, cut to $2,310.23 by the CDCC and saver's credit, leaving $2,089.77 refundable." +us,scenario_119,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It treated the filer as married filing jointly with a phase-out start of about $49,084, and used AGI of $55,800 without subtracting the traditional 401(k) and IRA amounts. As head of household the phase-out starts at $23,890 and AGI is $52,568.54, so the EITC is $1,276.30. The refundable CTC is $2,089.77, based on a $4,400 CTC, not $4,000." +us,scenario_119,federal_refundable_credits,qwen3.8-max,llm_error,asset_resource,False,"It denied the EITC because of $300,000 in stock, but the EITC has no asset test. It only limits investment income (about $12,000), and $800 of interest is well under that, so the EITC is $1,276.30. Its $565 ACTC is also too low compared with the $2,089.77 left after $2,310.23 of liability." us,scenario_119,free_school_meals_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -9657,116 +9781,118 @@ us,scenario_119,self_employment_tax,glm-5.3,parse_contract_failure,missing_outpu us,scenario_119,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,snap,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It correctly worked taxable income down to $39,469 and tax to $2,012 using the $1,560 child-care subtraction. It then dropped that result for an unexplained $2,296. It also never subtracted the 20% nonrefundable Virginia EITC ($152.85)." -us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"Its taxable income ($39,469) and pre-credit tax ($2,012) are right. It then treated the Virginia EITC as refundable only and never subtracted the 20% nonrefundable Virginia EITC ($152.85), which this output nets out." -us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It used the outdated $4,500 single standard deduction instead of $8,750, and left out both the $2,790 personal exemptions and the $1,560 child-care subtraction. It also applied no Virginia EITC. Even its own taxable income of $48,069 gives about $2,506 under Virginia's brackets, not $1,888." -us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,credit_phaseout,False,"It said Virginia tax was zero. After the $8,750 standard deduction, $2,790 exemptions and $1,560 child-care subtraction, taxable income is still $39,469, which produces $2,011.94 of tax. Only the $152.85 nonrefundable EITC reduces that." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of the 2026 figure of $8,750 and left out the $1,560 child-care subtraction, which gave $2,116. It then took off an unexplained 'small credit' of about $145 instead of the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction, which gave $2,116. It then raised the figure to $2,459 with no basis and never applied the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction, which gave $2,116. It then raised the figure to $2,280 for unexplained 'adjustments' instead of subtracting the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"Its taxable income ($39,469) and pre-credit tax ($2,011.97) match the reference. The whole gap comes from the federal EITC used for the 20% Virginia credit. The model phased the EITC out on AGI of $52,569, getting $1,276.20 and a $255.24 Virginia credit. The reference uses a federal EITC of $764.23 and a $152.85 Virginia credit, which corresponds to phasing out on $55,000 of gross wages." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used a $9,000 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction, which gave taxable income of $40,779 and tax of $2,087. It also said no nonrefundable credit applied, missing the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It left out the $2,790 personal exemptions and the $1,560 child-care subtraction, taxing $43,819 to get $2,262.60. It then subtracted an invented $220 low-income credit and a Virginia child-care credit that does not exist, since Virginia gives child care as a deduction. It also skipped the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"The $8,750 standard deduction and $2,790 exemptions are right. It left out the $1,560 child-care subtraction, so taxable income came out at $41,029 instead of $39,469. It also never subtracted the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting $6,389 of ESI premiums from the stated wages, used a made-up $9,470 standard deduction, and left out the exemptions and child-care subtraction. It then subtracted a $1,000 Virginia child tax credit, which is not part of Virginia's 2026 credits, instead of the 20% EITC." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction and left out both the $2,790 personal exemptions and the $1,560 child-care subtraction, taxing $44,069 instead of $39,469. It then applied a '40% of federal' Virginia child-care credit that does not exist, instead of the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (taxable income $41,279 instead of $39,469). It also never subtracted the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It used the outdated $4,500 standard deduction and left out the $1,560 child-care subtraction. It also got the brackets wrong, applying 5% to $17,000 and 5.75% only above $22,000; the correct brackets put 5% on $5,000–$17,000 and 5.75% on everything above. It applied no Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It understated taxable income at about $34,780 instead of $39,469 and subtracted a Virginia child-care credit that does not exist. It also took the nonrefundable EITC at 25% of an inflated federal EITC ($660), when Virginia's nonrefundable rate is 20% of a $764.23 federal EITC ($152.85)." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It started from an AGI of $51,713 instead of $52,569, and its $2,043 is above the correct pre-credit tax of $2,011.94. That means it left out the $1,560 child-care subtraction and never applied the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting ESI premiums and used a $3,000 standard deduction instead of $8,750; the two errors partly cancel. The lower AGI also inflated the federal EITC, so its 20% Virginia credit came out at $386.60 instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting ESI premiums, used $8,500 instead of $8,750, used $800 exemptions instead of $930, and left out the $1,560 child-care subtraction. It then subtracted a Virginia child-care credit that does not exist instead of the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave no derivation. The correct steps give $39,469 of taxable income and $2,011.94 of tax, less a $152.85 nonrefundable Virginia EITC. Its $1,974 is close to the pre-credit tax, which shows it skipped most of the EITC offset." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It cited a 'nonrefundable state child care credit' that Virginia does not have, since Virginia allows a $1,560 deduction for these expenses instead. It never applied the 20% nonrefundable EITC. Its $1,696.28 implies an understated taxable base compared with $39,469 taxed at $2,011.94." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It gave no derivation. Its $1,450.48 is $409 below the result of taxing $39,469 ($2,011.94) and subtracting the $152.85 EITC, which implies it understated taxable income or overstated nonrefundable credits." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It used an $8,000 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (tax $2,145 instead of $2,011.94). It then subtracted a Virginia child-care credit that does not exist and a $100 EITC instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It left out the $2,790 exemptions and the $1,560 child-care subtraction and used $8,500 for the standard deduction. It also set the federal EITC at a near-maximum $5,669.56, ignoring the 21.06% phase-out above about $23,890, so its 20% Virginia credit was $1,133.91 instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there is no answer to score." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It said it subtracted only the standard deduction, yet its $803.75 is less than half the correct figure. The correct steps give taxable income of $39,469, tax of $2,011.94, and $1,859.09 after the 20% EITC, so the model applied the rates to a far too small base." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It reported $180 of residual tax with no derivation. Virginia taxable income after the $8,750 standard deduction, $2,790 exemptions and $1,560 child-care subtraction is $39,469, giving $2,011.94 of tax. Only the $152.85 nonrefundable EITC offsets that." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"Its taxable income of $45,219 subtracts only about $7,350 from AGI. The standard deduction ($8,750), exemptions ($2,790) and child-care subtraction ($1,560) total $13,100, which gives $39,469. It also overstated the nonrefundable Virginia EITC at $267 instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"Its $1,483.02 implies taxable income of about $30,270, not the correct $39,469 (AGI $52,569 less $8,750, $2,790 and $1,560). It also described no 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"Its deductions and its $2,012 pre-credit tax are right. It said no nonrefundable credit was available and so never subtracted the 20% nonrefundable Virginia EITC ($152.85)." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child-care subtraction (tax $2,101.67 on $41,029 of taxable income). It then subtracted two $500 'child credits' that Virginia does not have, instead of the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 of ESI premiums from the stated gross wages. The correct AGI is $52,568.54 (wages plus interest less the 401(k) and IRA contributions), so its taxable income of $33,080 was $6,389 too low. It also never applied the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"It subtracted only the standard deduction and exemptions, leaving out the $1,560 child-care subtraction. It then took off a 'Virginia dependent-care credit' that does not exist instead of the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child-care subtraction (taxable income $41,029 instead of $39,469). It also never subtracted the 20% nonrefundable Virginia EITC ($152.85)." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting ESI premiums, which produced taxable income of $33,080. Virginia's brackets give $1,644.60 on that amount, so its $1,764.60 is also off by $120 in arithmetic. It never applied the 20% nonrefundable EITC." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"Its reasoning mentions only the standard deduction and exemptions, which alone give $2,101.67. What brings the correct answer to $1,859.09 is the $1,560 child-care subtraction and the 20% nonrefundable EITC, and the model applied neither." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (taxable income $41,279). It also never subtracted the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction and left out the $1,560 child-care subtraction, which gave a tentative tax of $2,116 instead of $2,011.94. It then took a $223 Virginia EITC instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"It did include the child-care subtraction and the exemptions, but used an $8,500 standard deduction instead of the 2026 figure of $8,750. That gave taxable income of $39,719 and tax of $2,026. It also subtracted a $260 nonrefundable Virginia EITC instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"Its taxable income of $35,330 is $4,139 below the correct $39,469 (AGI $52,569 less $8,750, $2,790 and $1,560). It also never applied the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"Its taxable income of $41,779 reflects an $8,000 standard deduction instead of $8,750 and leaves out the $1,560 child-care subtraction. It also took the nonrefundable Virginia EITC at about $238 instead of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there is no answer to score." -us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction. It also said no nonrefundable credit applied, missing the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It used a $5,200 standard deduction and mis-stated the brackets as 3% up to $17,000 and 5% above; Virginia actually taxes 5% on $5,000–$17,000 and 5.75% above that. It counted four $800 exemptions instead of three at $930. It then raised the tax after the exemptions instead of lowering it." -us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child-care subtraction (taxable income $41,279 instead of $39,469). It also said no nonrefundable credits applied, missing the 20% nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It added imputed employer health coverage to AGI ($59,111) instead of using $52,569, and mixed up the standard deduction with joint-filer and exemption figures. Its $1,173.89 also does not follow from its own taxable income of $41,111, which gives about $2,149 under Virginia's brackets." -us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"Its taxable income of $50,200 leaves out most of the $8,750 standard deduction and all of the $1,560 child-care subtraction, and its $2,024 base tax does not match Virginia's brackets on that amount. It then subtracted a $112 low-income credit this household does not qualify for, instead of the 20% nonrefundable EITC." -us,scenario_119,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Put 15% of an assumed ~$1,268 federal EITC into refundable credits. It ignored Virginia's rule that filers take either the refundable or the nonrefundable EITC. With about $2,000 of Virginia liability absorbing the credit, the EITC is claimed as nonrefundable and refundable credits are $0." -us,scenario_119,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Treated Virginia's EITC as a refundable 15% of the ~$1,276 federal EITC. Virginia requires a choice between refundable and nonrefundable. This household's roughly $2,000 state liability fully absorbs the credit, so it goes to the nonrefundable credits and state refundable credits are $0." -us,scenario_119,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Booked 20% of a federal EITC of ~$3,913 as a refundable Virginia credit. That federal figure is far too high: with AGI of ~$52.6k, the phaseout leaves about $1.3k. The model also missed that Virginia's EITC is taken in its nonrefundable form when state liability covers it, which leaves $0 of refundable state credits." -us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied 20% to a ~$1,431 federal EITC and reported the result as refundable. It never applied Virginia's either/or election between refundable and nonrefundable EITC. The household's ~$2,000 Virginia tax fully uses the nonrefundable credit, so refundable credits are $0." -us,scenario_119,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Computed a refundable Virginia EIC of 20% × $1,075.76. Virginia's EITC goes to the nonrefundable credits when state income tax (~$2,000 here) is large enough to absorb it. The EITC therefore reduces pre-refundable state tax, and state_refundable_credits is $0." -us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Reported a refundable Virginia EITC at 15% of an implied ~$2,216 federal EITC. It missed that Virginia filers take either the refundable or the nonrefundable EITC. Because this household's Virginia liability covers the credit, the credit is nonrefundable and refundable state credits are $0." -us,scenario_119,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no parseable value or explanation for state_refundable_credits. The correct answer is $0: Virginia's EITC is claimed as the nonrefundable credit against the household's roughly $2,000 state liability." -us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Put 20% of an understated $785 federal EITC (the correct figure is about $1.3k) into refundable credits. The core error is treating Virginia's EITC as automatically refundable. Virginia's liability absorbs the nonrefundable version, so refundable state credits are $0." -us,scenario_119,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Put 20% of an overstated $2,621.73 federal EITC into refundable credits. The federal EITC at ~$52.6k AGI is about $1.3k. More fundamentally, the model ignored that Virginia's EITC is claimed in nonrefundable form when state tax covers it, which makes state_refundable_credits $0." -us,scenario_119,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Estimated a refundable Virginia EITC of 20% × ~$1,246. The federal base is close, but Virginia makes the filer choose refundable or nonrefundable. With about $2,000 of state liability fully absorbing the credit, it is nonrefundable and refundable credits are $0." -us,scenario_119,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Reported 15% of a $1,474 federal EITC as a refundable Virginia credit. It missed the rule that Virginia's EITC is taken as a nonrefundable credit when the state liability (~$2,000 here) absorbs it, which leaves $0 in state refundable credits." -us,scenario_119,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Applied 20% to an overstated $2,623.83 federal EITC and called it refundable. It ignored Virginia's election between refundable and nonrefundable EITC. Here the credit is claimed against the roughly $2,000 state tax as nonrefundable, so refundable credits are $0." -us,scenario_119,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Reported 20% of a $1,300 federal EITC as a refundable Virginia credit. Virginia's EITC is claimed as the nonrefundable version when state liability covers it, and this household's ~$2,000 Virginia tax does. The credit therefore reduces pre-refundable state tax, and state_refundable_credits is $0." -us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for state_refundable_credits. The correct answer is $0 because Virginia's EITC is taken as a nonrefundable credit against the household's state liability. -us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Put 20% of an understated $764.23 federal EITC into refundable credits. The federal EITC is about $1.3k at this AGI. The decisive error is treating Virginia's EITC as refundable: the household's ~$2,000 Virginia tax absorbs the nonrefundable form, so refundable state credits are $0." +us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model correctly worked out $39,469 of taxable income and $2,012 of tax with the $1,560 child care deduction, then dropped that result for an unsupported $2,296. It also never subtracted the nonrefundable Virginia EITC of 20% × $1,276.30 = $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model got the $2,012 pre-credit tax right but wrongly treated the Virginia EITC as only refundable. Here Virginia's nonrefundable 20% credit ($255.26, from the $1,276.30 federal EITC) applies before refundable credits, which brings tax to $1,756.68." +us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an outdated $4,500 standard deduction instead of $8,750. It also left out the $2,790 of personal exemptions, the $1,560 child care deduction and the $255.26 nonrefundable Virginia EITC. Its $1,888 doesn't even match its own $48,069 taxable income." +us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model claimed zero Virginia tax. After the $8,750 standard deduction, $2,790 of exemptions and the $1,560 child care deduction, $39,468.54 of income is still taxable. That produces $2,011.94 of tax, and only $255.26 of it is offset by the nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child care deduction, getting $2,116 of tax. It then subtracted unspecified 'small nonrefundable credits' instead of the 20% Virginia EITC of $255.26 on the correct $2,011.94." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model worked out $2,116 using an $8,500 standard deduction and no $1,560 child care deduction, then raised it to $2,459 for no stated reason. It also never applied the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child care deduction. It then raised its $2,116 with unexplained 'adjustments' instead of subtracting the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model used a $9,000 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction. It also said no Virginia nonrefundable credit applies, missing the 20% EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model dropped the $2,790 of personal exemptions and the $1,560 child care deduction. It then made up a $220 low-income adjustment and a 30% child care credit. Virginia handles child care as a deduction, and the only nonrefundable credit here is the $255.26 EITC (20% of the federal EITC)." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model stopped at $41,029 of taxable income. It never subtracted the $1,560 child and dependent care expense deduction (giving $39,468.54 and $2,011.94 of tax) or the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,389 ESI premiums again, getting $46,180 of AGI, even though federal AGI is $52,568.54. It then made up a $1,000 Virginia child tax credit. Virginia's only nonrefundable credit here is the 20% EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model left out the $2,790 of personal exemptions and the $1,560 child care deduction, and used $8,500 instead of the $8,750 standard deduction. It also applied a made-up 40%-of-federal child care credit instead of the 20% Virginia EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750, left out the $1,560 child care deduction, and applied no nonrefundable Virginia EITC. That EITC is $255.26 (20% of $1,276.30) and would reduce the $2,011.94 tax to $1,756.68." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used an outdated $4,500 standard deduction instead of $8,750 and wrongly applied the 5% bracket to $17,000 instead of $12,000. It also left out the $1,560 child care deduction and the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the ESI premiums again, which lowered AGI below the correct $52,568.54. It made up a 25% Virginia child care credit; Virginia gives a $1,560 deduction instead. It also took 25% of an overstated federal EITC, when the Virginia credit is 20% of $1,276.30 = $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model started from a miscomputed $51,713 of AGI instead of $52,568.54. Its $2,043 is close to the pre-credit tax, which means it never subtracted the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,389 ESI premiums again, getting $46,180 of AGI instead of $52,568.54. It also used a $3,000 standard deduction instead of $8,750. Because of the lower AGI, its 20% Virginia EITC came out at $386.60 instead of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the ESI premiums again ($46,180 of AGI) and used outdated $800 exemptions and an $8,500 standard deduction. It replaced the $1,560 child care deduction with a made-up child care credit and never applied the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The $1,974 answer is close to Virginia's pre-credit tax and leaves out the nonrefundable 20% Virginia EITC. The correct path is $2,011.94 of tax on $39,468.54 of taxable income, less a $255.26 EITC, for $1,756.68." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model subtracted a 'state child care credit' that Virginia doesn't have; Virginia allows the $1,560 as a deduction from taxable income. It also left out the actual nonrefundable credit, the 20% Virginia EITC of $255.26 on $2,011.94 of tax." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The $1,450.48 answer is $306 below the correct result. The correct path is $2,011.94 of tax on $39,468.54 of taxable income, minus only the $255.26 nonrefundable Virginia EITC. Getting $1,450.48 requires credits or deductions beyond those Virginia allows here." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model used an $8,000 standard deduction instead of $8,750 and left out the $1,560 child care deduction. It added a made-up $78 Virginia child care credit and understated the 20% Virginia EITC as $100 instead of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model dropped the $2,790 of exemptions and the $1,560 child care deduction and used $8,500 instead of $8,750. It also put the federal EITC at $5,669.56 without applying the phase-out above AGI of about $24,000. At $52,568.54 of AGI the federal EITC is only $1,276.30, so the Virginia credit is $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no answer for state_income_tax_before_refundable_credits, so there was no value to score." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model's $803.75 is less than half the correct tax. Virginia taxable income is $39,468.54 after the $8,750 standard deduction, $2,790 of exemptions and $1,560 of child care. That produces $2,011.94 of tax, reduced only by the $255.26 nonrefundable EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model's $180 assumes nonrefundable credits wipe out almost all the tax. The only Virginia nonrefundable credit here is the 20% EITC of $255.26, which leaves $1,756.68 of the $2,011.94 tax." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model listed the standard deduction, exemptions and child care deduction but reported $45,219 of taxable income instead of $39,468.54, which subtracts only $7,350. It then slightly overstated the Virginia EITC as $267 instead of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied the standard deduction, exemptions and child care subtraction but reported $1,483.02, which is $528.92 below the $2,011.94 pre-credit tax. The only nonrefundable reduction Virginia allows here is the $255.26 EITC (20% of $1,276.30)." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model got the $2,012 pre-credit tax right but said no nonrefundable credit applied. It missed the Virginia EITC at 20% of the $1,276.30 federal EITC ($255.26), which brings tax to $1,756.68." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model made up two $500 nonrefundable Virginia child credits, which Virginia doesn't have. It also left out the $1,560 child care deduction and missed the actual nonrefundable credit, the 20% EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model subtracted the $6,389 ESI premiums again, getting $46,180 of AGI; those premiums are already excluded from the $55,000 of wages, so AGI is $52,568.54. It also never subtracted the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model treated child care as a nonrefundable Virginia credit (about $62) instead of a $1,560 deduction from taxable income. It also left out the nonrefundable Virginia EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model stopped at $41,029 of taxable income. It left out the $1,560 child and dependent care expense deduction (taxable income should be $39,468.54 and tax $2,011.94) and the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted the ESI premiums again, getting $46,180 of AGI and $33,080 of taxable income. It then misapplied the brackets: tax on $33,080 is $1,644.60, not $1,764.60. It also never applied the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model's $1,874 doesn't follow the correct steps: $39,468.54 of taxable income after the $1,560 child care deduction gives $2,011.94 of tax, less the $255.26 nonrefundable Virginia EITC. Its figure shows it either missed the child care deduction or misstated the 20% EITC." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child care deduction. It also never applied the 20% nonrefundable Virginia EITC ($255.26), so it stopped at $2,116." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model left out the $1,560 child and dependent care expense deduction and used $8,500 instead of the $8,750 standard deduction, getting $2,116 of tax instead of $2,011.94. It also understated the 20% Virginia EITC as $223 instead of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used the 2025 $8,500 standard deduction instead of the 2026 $8,750, which overstated taxable income by $250 ($39,719 vs. $39,468.54). It also slightly overstated the 20% Virginia EITC as $260 instead of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model taxed $35,330 instead of the correct $39,468.54 of Virginia taxable income ($52,568.54 − $8,750 − $2,790 − $1,560). It also never subtracted the $255.26 nonrefundable Virginia EITC from the resulting $2,011.94." +us,scenario_119,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an $8,000 standard deduction instead of $8,750 and left out the $1,560 child care deduction, getting $41,779 of taxable income instead of $39,468.54. It also understated the Virginia EITC as $238 instead of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no answer for state_income_tax_before_refundable_credits, so there was no value to score." +us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child care deduction. It also said no nonrefundable credit applied, missing the 20% Virginia EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a made-up $5,200 standard deduction, wrong bracket widths and four $800 exemptions. It then moved its tax upward after subtracting exemptions, ending at $2,867. The correct steps give $39,468.54 of taxable income, $2,011.94 of tax and a $255.26 EITC." +us,scenario_119,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model used a $3,000 standard deduction instead of the 2026 $8,750 and left out the $1,560 child care deduction. It also never applied the $255.26 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child care deduction. It also said no nonrefundable credit applied, missing the 20% Virginia EITC of $255.26." +us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model added imputed employer health coverage to AGI, getting $59,111, when the correct federal AGI is $52,568.54. It also made up an $18,000 joint standard deduction and gave a final tax that doesn't match its own taxable income." +us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model reported $50,200 of taxable income, which ignores most of the $8,750 standard deduction and the $1,560 child care deduction. It also made up a $112 credit for taxable income under $50,250 instead of applying the 20% Virginia EITC of $255.26." +us,scenario_119,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Counted 15% of the federal EITC (about $190) as a refundable Virginia credit without applying Virginia's either/or choice between EITC forms. The roughly $2,000 Virginia liability fully absorbs the larger 20% nonrefundable credit, so that is the form claimed and the refundable credit is $0." +us,scenario_119,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Reported 15% of a $1,276 federal EITC ($191) as a refundable state credit, ignoring that filers claim either the refundable or the nonrefundable VA EITC, not both. Virginia tax of about $2,000 fully absorbs the 20% nonrefundable credit, so that one is chosen and state refundable credits are $0." +us,scenario_119,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Treated the 20% VA EITC as refundable, but 20% is the rate of the nonrefundable option. Virginia tax liability fully absorbs that credit, so it reduces tax before refundable credits and the refundable credit is $0. The $3,913 federal EITC base is also too high, because AGI of about $52.6k puts the household deep in the phase-out range." +us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Called the 20% VA EITC ($286.20) refundable. That rate belongs to the nonrefundable credit, which this household elects because its roughly $2,000 Virginia liability absorbs it fully. Nothing is claimed through the refundable pathway, so the correct value is $0." +us,scenario_119,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Reported 20% of the federal EITC ($215.15) as a refundable Virginia credit, but that amount is the nonrefundable EITC. With Virginia tax far above it, the household takes the nonrefundable credit, and state refundable credits are $0." +us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Applied the 15% refundable VA EITC ($332.46) without checking the either/or election against the nonrefundable credit. The roughly $2,000 Virginia liability fully absorbs the 20% nonrefundable credit, which makes it the better choice and leaves refundable credits at $0." +us,scenario_119,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits. The correct derivation is $0, because the household elects Virginia's nonrefundable EITC, which its state liability fully absorbs." +us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Counted 20% of a $785 federal EITC ($157) as refundable, but that 20% credit is the nonrefundable VA EITC. The household's Virginia tax of about $2,000 absorbs it fully, so the refundable credit is $0." +us,scenario_119,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Treated the 20% VA EITC ($524.35) as refundable, missing that filers choose either the nonrefundable or the refundable form. Because Virginia liability exceeds the credit, the nonrefundable form is claimed and state refundable credits are $0." +us,scenario_119,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Put 20% of the federal EITC ($249) into refundable credits, but that is the nonrefundable VA EITC. The household's roughly $2,000 Virginia tax absorbs it fully, so no refundable credit is claimed and the value is $0." +us,scenario_119,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Computed the refundable VA EITC at 15% ($221.10) without comparing it to the 20% nonrefundable alternative. Virginia tax is far above both amounts, so the nonrefundable credit is chosen and refundable credits are $0." +us,scenario_119,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Called the 20% VA EITC ($524.77) refundable, but at this income it is claimed as the nonrefundable credit against roughly $2,000 of Virginia liability. The refundable form is not elected, so state refundable credits are $0." +us,scenario_119,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Treated 20% of the federal EITC ($260) as refundable, ignoring Virginia's either/or choice between EITC forms. The household's Virginia tax absorbs the 20% nonrefundable credit in full, so that is the form taken and the refundable value is $0." +us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits. The correct derivation is $0, because the household elects Virginia's nonrefundable EITC, which its state liability fully absorbs." +us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Reported 20% of a $764.23 federal EITC ($152.85) as refundable, but that is the nonrefundable VA EITC. With roughly $2,000 of Virginia tax to offset, the household claims it as a nonrefundable credit, so refundable credits are $0." +us,scenario_119,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Counted 20% of the federal EITC ($219) as a refundable Virginia credit without applying the either/or election. The household's Virginia tax absorbs the nonrefundable credit in full, so it is the form claimed and state refundable credits are $0." us,scenario_119,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached about $200,500 of taxable income but reported $30,238. The 2026 single brackets give about $40,700 on that amount. On the deduction side it also used the old $10,000 SALT cap instead of the full $13,908 allowed under the $40,400 OBBBA cap, took the full $3,744 of mortgage interest instead of $2,808 under the $750,000 debt limit, skipped the 0.5% charitable floor, and left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI and skipped the $28 IRA deduction, which put taxable income at $198,819 instead of $196,918. It also figured NIIT as about $70 on $1,853 after netting the farm rent loss. The correct NIIT is 3.8% of the full $4,300 of interest and rental income, or $163.40." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It counted 100% of Social Security as income and took a $30,000 standard deduction instead of itemizing $35,717 (SALT $13,908, mortgage interest $2,808, charity $19,001). It then subtracted a made-up $11,297 of withholding, which never reduces tax before refundable credits. Its bracket tax of $29,119 on $210,670 is also far below the roughly $43,900 that the 2026 single schedule produces." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a $25,000 guess without doing any bracket calculation. AGI of $232,635 minus $35,717 of itemized deductions leaves $196,918 of taxable income, which is taxed at $39,858 under the 2026 single brackets, plus $163.40 of NIIT. The guess is about $15,000 too low." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own bracket arithmetic came to $40,823, but it then reported $38,740 with no credit or adjustment to explain the cut. Its deductions capped SALT at the old $10,000 instead of the full $13,908 allowed under OBBBA's $40,400 cap. It also skipped the $750,000 mortgage-debt limit and the 0.5% charitable floor, and left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at the expired $10,000 limit instead of deducting the full ~$13,908 of CT income tax and property tax under OBBBA's $40,400 cap. It also skipped the 0.5%-of-AGI charitable floor and added the $1,750 refund to AGI, which left taxable income at $202,030 instead of $196,918. It also left out the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the $2,012 property tax as SALT and left out the ~$11,896 of CT income tax. It also took the full $3,744 of mortgage interest instead of $2,808 under the $750k/$1M debt ratio and skipped the 0.5% charitable floor, which produced $208,465 of taxable income instead of $196,918. Its $40,650 is also below the 2026 bracket tax on its own taxable income (about $43,165), and it left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI ($234,385 instead of $232,635). That raised taxable income to $198,906 instead of $196,918 and added about $480 of regular tax. It also figured NIIT as $70 on $1,853 after netting the farm rent loss, when the correct NIIT is 3.8% of the $4,300 of interest and rental income, or $163.40." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied pre-2018 rates (10/15/25/28/33%), a personal exemption, no SALT cap, and 2%-floor miscellaneous deductions. OBBBA made the TCJA brackets and the repeal of those items permanent. It also left CT income tax out of SALT, and its submitted $42,046 does not match its own final figure of $47,383." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached $207,651 of taxable income but reported $24,500. The 2026 single brackets give about $42,900 on that amount. Its taxable income is too high because it left the ~$11,896 of CT income tax out of SALT, even though it is fully deductible under the $40,400 cap. It also left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $235,031 includes the $1,750 state refund and does not take the $617 traditional 401(k) deferral out of wages; the correct AGI is $232,635. That pushed taxable income to $199,649 instead of $196,918. It also figured NIIT as $70 on $1,853 net instead of $163.40 on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer-sponsored insurance premiums from wages, but employment income is $164,980, reduced only by the 401(k) deferral. It also skipped the 0.5% charitable floor and understated SALT at $11,815. Its $34,322 is about $4,400 below the 2026 bracket tax on its own $192,198 of taxable income, and it left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied pre-2018 brackets, a $5,063 personal exemption, and 2%-floor miscellaneous deductions. OBBBA made the TCJA rate schedule permanent and kept the personal exemption and miscellaneous itemized deductions repealed. It also subtracted the $7,429 of ESI premiums from wages and computed NIIT on $1,853 instead of $4,300." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used a $5,250 personal exemption with pre-TCJA brackets. OBBBA keeps the personal exemption at zero and the 10/12/22/24% schedule permanent. Its $42,059 of itemized deductions also exceeds the allowable $35,717, and it figured NIIT on $1,853 net instead of the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state refund to AGI ($234,414 instead of $232,635) and deducted the full $20,164 of charitable gifts without OBBBA's 0.5%-of-AGI floor, which allows $19,001. That gave $198,557 of taxable income instead of $196,918. It also left out the $163.40 net investment income tax on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, so it dropped the SALT cap, deducted $2,966 of 2%-floor miscellaneous employee expenses, took the full $3,744 of mortgage interest, and used pre-TCJA brackets. OBBBA made the TCJA brackets, the $750,000 mortgage limit, and the repeal of miscellaneous deductions permanent. The higher pre-TCJA rates produced $45,093, versus $39,858 of regular tax plus $163.40 of NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no calculation. The correct path is AGI of $232,635 minus $35,717 of itemized deductions, or $196,918 of taxable income, taxed at $39,858 under the 2026 single brackets, plus $163.40 of NIIT. Its $39,591 is $431 short, consistent with leaving out the NIIT and overstating deductions by about $1,100." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages and deducted 2%-floor miscellaneous employee expenses, which OBBBA permanently repealed. It then applied bracket rates heavier than the 2026 OBBBA schedule, producing $41,404 of regular tax on a taxable base below the correct $196,918. Its $87 of NIIT also falls short of 3.8% × $4,300 = $163.40." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied a $5,000 personal exemption, $3,149 of 2%-floor miscellaneous deductions, uncapped mortgage interest, and pre-TCJA rate brackets. None of these apply under OBBBA's permanent extension. It also subtracted the $7,429 of ESI premiums from wages and figured NIIT as $70 on $1,853 instead of $163.40 on $4,300." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $33,814 is about $6,200 short, consistent with understating taxable income by about $25,000 at the 24% marginal rate and leaving out the NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages, giving AGI of $226,985 instead of $232,635. It also capped SALT at the old $10,000 instead of the ~$13,908 allowed under OBBBA's $40,400 cap and skipped the 0.5% charitable floor. It left out the $163.40 NIIT on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $37,824 is $2,198 short, consistent with taxable income about $8,500 too low and leaving out the NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $33,024 is about $7,000 short, consistent with understating taxable income by roughly $28,500 and leaving out the NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the $2,012 property tax as SALT and left out the ~$11,896 of CT income tax, which is fully deductible under OBBBA's $40,400 cap. It also skipped the 0.5% charitable floor, so itemized deductions came to $24,984 instead of $35,717 and taxable income to $209,430 instead of $196,918. It also added the $1,750 refund to AGI and left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It double-counted mortgage interest at $7,488 by adding the head's $3,744 to the tax unit's $2,976 + $768, and it ignored the $750,000 debt limit that cuts deductible interest to $2,808. It also capped SALT at the old $10,000 instead of ~$13,908 under the $40,400 cap and skipped the 0.5% charitable floor. It left out the $163.40 NIIT on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It did no calculation and treated deductions as limited by the standard deduction, even though itemized deductions of $35,717 are far above the $18,150 standard deduction. The correct $196,918 of taxable income gives $39,858 of regular tax plus $163.40 of NIIT. Its $38,141 is consistent with taxable income about $7,200 too low and leaving out the NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no actual calculation. The correct path is AGI of $232,635 minus $35,717 of itemized deductions, or $196,918 of taxable income, taxed at $39,858, plus $163.40 of NIIT. Its $35,233 is consistent with understating taxable income by about $19,000 and leaving out the NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired: a $5,300 personal exemption, about $40,615 of itemized deductions including miscellaneous expenses, and pre-TCJA brackets. OBBBA made the TCJA schedule permanent, with no personal exemption and no miscellaneous itemized deductions. Its NIIT of about $70 also falls short of $163.40 on $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $210,286 is $13,368 above the correct $196,918. That gap matches leaving the ~$11,896 of CT income tax out of the SALT deduction, which is fully deductible under OBBBA's $40,400 cap. Its $43,748 also leaves out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It figured NIIT as about $70 on investment income netted against the farm rent loss, instead of 3.8% of the full $4,300 of interest and rental income ($163.40). With AGI of $234,414, which includes the $1,750 refund, its regular tax of about $39,594 implies itemized deductions near $38,600. That is about $2,900 above the allowable $35,717 (SALT $13,908, mortgage interest $2,808, charity $19,001 after the 0.5% floor)." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, restored the personal exemption, and applied pre-TCJA rates. OBBBA made the 10/12/22/24% schedule and the zero personal exemption permanent. Its $48,134 is about $8,100 above the correct $39,858 of regular tax plus $163.40 of NIIT on $196,918 of taxable income." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer-sponsored insurance premiums from wages, giving AGI of $226,985 instead of $232,635; employment income is $164,980, reduced only by the 401(k) deferral. That lowered taxable income by about $5,650. It also left out the $163.40 NIIT owed because MAGI is above $200,000." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It claimed $40,221 of itemized deductions, about $4,500 more than the allowable $35,717 (SALT $13,908, mortgage interest $2,808 after the debt limit, charity $19,001 after the 0.5% floor). It also added the $1,750 refund to AGI. It left out the $163.40 NIIT on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $234,414, which adds the $1,750 refund, and itemized deductions of about $38,271, roughly $2,550 above the allowable $35,717. That put taxable income at $196,143 instead of $196,918. Its bracket arithmetic is right for that figure, but it left out the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages, putting AGI at $226,985 instead of $232,635 and taxable income at $190,999 instead of $196,918. It also left out the $163.40 NIIT on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct calculation is $196,918 of taxable income, $39,858 of bracket tax, and $163.40 of NIIT. Its $39,250 is $772 short, consistent with leaving out the NIIT and understating taxable income by about $2,500." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied a $5,427 personal exemption, $2,966 of 2%-floor miscellaneous deductions, uncapped mortgage interest, and pre-TCJA brackets. OBBBA made the TCJA schedule, the $750,000 mortgage limit, and the repeal of the exemption and miscellaneous deductions permanent. It also left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied restored 10/15/25/28% brackets and a $5,437 personal exemption on the assumption that the TCJA expired in 2026. OBBBA made the 10/12/22/24% schedule and the zero personal exemption permanent. It also left out the $163.40 NIIT on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $20,164 of charitable gifts without OBBBA's 0.5%-of-AGI floor, which allows $19,001, and overstated SALT at $15,371 instead of $13,908. Together these cut taxable income to $194,321 instead of $196,918. It also figured NIIT on $1,853 after netting the farm rent loss instead of on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied pre-TCJA brackets, $2,966 of 2%-floor miscellaneous deductions, and uncapped mortgage interest. OBBBA made the TCJA rate schedule, the $750,000 mortgage limit, and the repeal of miscellaneous deductions permanent. It also left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $2,966 of unreimbursed employee business expenses as a 2%-floor miscellaneous itemized deduction, which OBBBA permanently eliminated, and it skipped the 0.5% charitable floor. Together these cut taxable income to $194,235 instead of $196,918. Its NIIT of $163 is correct, but its regular tax is about $640 too low." -us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $40,021.82 result." -us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $40,021.82 result." -us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported zero federal tax even though taxable income is $196,918, which produces $39,858 of regular tax plus $163.40 of NIIT. No deduction or nonrefundable credit removes that liability." -us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $3,744 of mortgage interest without applying the $750,000/$1,000,000 acquisition-debt ratio, which limits it to $2,808. It also added the $1,750 refund to AGI, leaving taxable income at $197,685 instead of $196,918. It left out the $163.40 NIIT owed on the $4,300 of interest and rental income because MAGI is above $200,000." -us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took an $18,525 standard deduction without comparing it to the $35,717 of itemized deductions (SALT $13,908, mortgage interest $2,808, charity $19,001). It also added the $1,750 refund, so taxable income came to $214,506 instead of $196,918. It used 2025 bracket thresholds, taxed the amount above $197,300 at 24% instead of 32%, and left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It treated the head as married filing separately instead of single and added the $6,147 of tax-exempt pension to taxable pensions. It stacked charitable donations on top of the standard deduction instead of itemizing, and claimed a $750 retirement income credit that does not exist. Its submitted $48,557 also contradicts its own figure of $45,287 − $750 = $44,537." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its deductions used the expired $10,000 SALT cap, the full $3,744 of mortgage interest (should be $2,808) and no 0.5% charitable floor, giving about $200,500 of taxable income. The main error is the bracket math: 2026 single brackets on about $200,500 give roughly $40,700, not $30,238. It also left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI and skipped the IRA deduction, getting $234,414 instead of $232,635, which pushed taxable income to $198,819 instead of $196,918. It also netted the farm rent loss into NIIT (about $70 on $1,853) instead of taking 3.8% of the full $4,300 of interest and rent ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used an invented $30,000 standard deduction instead of itemizing $35,717, and it taxed all $41,900 of Social Security instead of 85%. It then subtracted $11,297 of 'estimated withholding', which is a tax payment, not a nonrefundable credit, so the result fell far below $40,021.82." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a $25,000 guess with no computation. The correct path is AGI $232,635, minus $35,717 itemized, for $196,918 taxable, which gives $39,858 of tax plus $163.40 NIIT. The guess undercounts the 24%-bracket liability by about $15,000." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It computed $40,823 of bracket tax and then reported $38,740 with no stated reduction. Its deductions used the old $10,000 SALT cap instead of the $13,908 allowed under the $40,400 cap, the full mortgage interest instead of $2,808, and no charitable floor. It also added the $1,750 refund to AGI and left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at the expired $10,000 instead of deducting $13,908 under the 2026 $40,400 cap, and it skipped the 0.5% charitable floor. It also added the $1,750 refund to AGI and never subtracted the $617 401(k) deferral, reaching $202,030 of taxable income instead of $196,918. It left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It left about $11,896 of Connecticut income tax out of SALT and applied neither the $750k/$1M mortgage-debt limit nor the 0.5% charitable floor. That gave $25,920 of itemized deductions instead of $35,717 and $208,465 of taxable income instead of $196,918. It also left out the $163.40 NIIT and understated bracket tax on its own taxable income." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI ($234,385 instead of $232,635), which raised taxable income to $198,906 instead of $196,918. It netted the farm rent loss into NIIT ($70 on $1,853) instead of applying 3.8% to the full $4,300 of interest and rent ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025 and applied pre-TCJA 10/15/25/28/33% brackets, a personal exemption, uncapped SALT and 2%-floor miscellaneous deductions. OBBBA made the TCJA rate schedule, the $750k mortgage limit and the repeal of exemptions and miscellaneous deductions permanent. It also left CT income tax out of SALT, and its final $42,046 does not match any of its own computations." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached the correct $232,635 AGI but left CT income tax out of SALT and skipped the 0.5% charitable floor, so itemized deductions were $24,984 instead of $35,717. The tax step then failed: 2026 single brackets on its own $207,651 of taxable income give about $43,000, not $24,500. It also left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $235,031 includes the $1,750 state refund and never subtracts the $617 traditional 401(k) deferral or the IRA deduction. That gave $199,649 of taxable income instead of $196,918. It set NIIT at $70 on income netted with the farm loss, instead of $163.40 on the $4,300 of interest and rent." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer insurance premiums from gross wages; wages should be $165,597 less only the 401(k) deferral. It also added the $1,750 refund and skipped the charitable floor. Even on its own $192,198 of taxable income, 2026 brackets give about $38,700, so the $34,322 also contains a bracket error, and it left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA rules: a $5,063 personal exemption, pre-TCJA brackets and a 2%-floor miscellaneous deduction. OBBBA permanently kept the TCJA rate schedule and removed exemptions and miscellaneous deductions. It also subtracted the $7,429 of ESI premiums from wages, deducted mortgage interest with no limit, and computed NIIT on $1,853 instead of $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It treated 2026 as a post-TCJA-sunset year, with a $5,250 personal exemption, pre-TCJA brackets and inflated itemized deductions of $42,059. OBBBA made the 10/12/22/24/32/35/37% schedule permanent with no personal exemption. It also netted the farm loss into NIIT instead of taxing the full $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 refund to AGI and skipped the IRA deduction ($234,414 instead of $232,635). It deducted the full $20,164 of charity without the 0.5% floor, giving $198,557 of taxable income instead of $196,918. It left out the $163.40 NIIT entirely." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset: no SALT cap, $2,966 of unreimbursed employee expenses taken as a 2%-floor miscellaneous deduction, the full $3,744 of mortgage interest, and higher pre-TCJA brackets. Under OBBBA those TCJA provisions are permanent, so the correct answer is $39,858 on $196,918 of taxable income plus $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. The correct figure is $39,858.42 of bracket tax on $196,918 of taxable income plus $163.40 NIIT. Its $39,591 is $431 short, which fits leaving out NIIT and taking about $1,100 of extra deductions, such as skipping the 0.5% charitable floor." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from gross wages, added the $1,750 refund, and deducted miscellaneous employee expenses, which OBBBA permanently eliminated. Its $41,404 of regular tax on a lower AGI reflects a heavier bracket schedule than the 2026 single schedule. It set NIIT at $87 instead of $163.40 on $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It treated 2026 as a post-TCJA year: a $5,000 personal exemption, uncapped SALT, a $3,149 miscellaneous deduction, full mortgage interest and pre-TCJA brackets. It also subtracted the $7,429 of ESI premiums from wages. OBBBA made the TCJA rates, the $750k mortgage limit and the elimination of exemptions and miscellaneous deductions permanent." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct path is AGI $232,635, $35,717 itemized and $196,918 taxable, which gives $39,858.42 of tax plus $163.40 NIIT. Its $33,814 is about $6,200 short, matching a bracket computation that undertaxes the 24% band." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from wages ($226,985 AGI instead of $232,635) and capped SALT at the old $10,000 instead of the $13,908 allowed under the $40,400 cap. It also skipped the charitable floor and left out the $163.40 NIIT. Its bracket math overstated tax on its own $194,013 of taxable income, which partly offset these errors." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct computation is $35,717 of itemized deductions, $196,918 of taxable income, and $39,858.42 of tax plus $163.40 NIIT. Its $37,824 is $2,198 short, which equals about $8,500 of extra deductions taxed at 24% on top of the allowed itemized amounts." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct computation is $35,717 of itemized deductions, $196,918 of taxable income, and $39,858.42 of tax plus $163.40 NIIT. Its $33,023.76 is about $7,000 short, matching a bracket schedule that badly undertaxes income in the 24% band." +us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left about $11,896 of Connecticut income tax out of SALT and skipped the 0.5% charitable floor, so itemized deductions were $24,984 instead of $35,717. It also added the $1,750 refund to AGI. Taxable income of $209,430 instead of $196,918 pushed income into the 32% bracket, and it left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It double-counted mortgage interest at $7,488; the head's $3,744 is the same interest as the tax unit's $2,976 + $768. It also ignored the $750k/$1M debt limit, which cuts the deduction to $2,808. It capped SALT at the old $10,000 instead of the $13,908 allowed under the $40,400 cap, skipped the charitable floor, and left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation and wrongly said itemized deductions are limited by the standard deduction. The household itemizes $35,717 against an $18,150 standard deduction, which gives $39,858.42 of tax plus $163.40 NIIT. Its $38,141 is $1,881 short." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It treated mortgage interest and SALT as 'nonrefundable credit-like items' when they are itemized deductions, and it showed no computation. Correctly itemizing $35,717 gives $196,918 of taxable income and $39,858.42 of tax plus $163.40 NIIT. Its $35,233 is about $4,800 short." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: a $5,300 personal exemption, pre-TCJA brackets and inflated itemized deductions of $40,615. OBBBA kept the TCJA rate schedule and removed personal exemptions. The correct answer is $39,858 on $196,918 plus NIIT of $163.40 on $4,300, not $70." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $43,747.57 is the correct 2026 bracket tax on $210,286 of taxable income, but that taxable income is about $13,400 too high. It left about $11,896 of Connecticut income tax out of the SALT deduction. It also left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 refund to AGI ($234,414). It netted the farm loss into NIIT (about $70) instead of taking $163.40 on the full $4,300 of interest and rent. Its bracket tax implies about $195,800 of taxable income, below the correct $196,918, so it itemized more than the allowable $35,717." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules, with a restored personal exemption and pre-TCJA rates. OBBBA made the TCJA schedule permanent with no personal exemption. Even its $203,194 of taxable income gives only about $41,500 under 2026 single brackets, so the extra comes from the pre-TCJA rate structure. It also left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer insurance premiums from gross wages, getting $226,985 of AGI instead of $232,635; wages are $165,597 less only the 401(k) deferral. It also left out the $163.40 NIIT on $4,300 of interest and rent, which applies because MAGI is over $200,000." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It itemized $40,221 instead of $35,717, overstating the SALT and charitable amounts. The allowed amounts are $13,908 of SALT and $19,001 of charity after the 0.5% floor. It also added the $1,750 refund to AGI, and its $39,204 is the bracket tax on $194,193 with no $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $39,672.32 is the correct 2026 bracket tax on $196,143 of taxable income, but it left out the 3.8% NIIT ($163.40) on $4,300 of interest and rent, which applies because MAGI is over $200,000. It also added the $1,750 refund to AGI and overstated itemized deductions (about $38,271 instead of $35,717)." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of ESI premiums from gross wages, getting $226,985 of AGI instead of $232,635, and it left out the $163.40 NIIT. Its $38,437.74 is the correct bracket tax on its understated $190,999 of taxable income." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct computation is $35,717 itemized, $196,918 taxable and $39,858.42 of tax plus $163.40 NIIT. Its $39,250 is $772 short, matching an omitted NIIT plus about $2,500 of extra deductions taxed at 24%." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: a $5,427 personal exemption, a $2,966 miscellaneous employee-expense deduction, the full $3,744 of mortgage interest and pre-TCJA brackets. OBBBA keeps the TCJA 2026 schedule with no exemption or miscellaneous deductions, so the correct answer is $39,858 plus $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied restored pre-TCJA 10/15/25/28% brackets and a $5,437 personal exemption, treating 2026 as post-sunset. OBBBA made the 10/12/22/24/32% schedule permanent and removed exemptions. It also left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It overstated SALT at $15,371; CT income tax is about $11,896, for $13,908 of total SALT. It skipped the 0.5% charitable floor ($19,001 deductible, not $20,164), which cut taxable income to $194,321 instead of $196,918. It netted the farm loss into NIIT ($70 on $1,853) instead of taking $163.40 on $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-TCJA brackets and deducted $2,966 of miscellaneous employee expenses and the full $3,744 of mortgage interest. OBBBA permanently kept the TCJA rates, the $750k mortgage limit and the repeal of miscellaneous itemized deductions. It also added the refund to AGI and left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $2,966 of unreimbursed employee expenses as a miscellaneous itemized deduction, which OBBBA permanently repealed. It also skipped the 0.5% charitable floor, so itemized deductions were about $40,150 instead of $35,717 and taxable income was $194,235 instead of $196,918. Its $163 NIIT was correct." +us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against $40,021.82." +us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against $40,021.82." +us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $0 with no derivation, even though the single filer has $232,635 of AGI. After $35,717 of itemized deductions, $196,918 of taxable income gives $39,858.42 of bracket tax plus $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"It itemized only charity, property tax and limited mortgage interest (about $23,800), leaving out about $11,896 of Connecticut income tax, which belongs in SALT under the $40,400 cap. That pushed taxable income to about $211,000 instead of $196,918 and into the 32% bracket, and it left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It did not apply the $750k/$1M debt limit to mortgage interest ($3,744 instead of $2,808), and it added the $1,750 refund to AGI. It also left out the $163.40 NIIT on $4,300 of interest and rent despite MAGI over $200,000. The higher taxable income ($197,685) and the missing NIIT nearly cancel, leaving it $20 high." +us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took an $18,525 standard deduction instead of itemizing $35,717 ($13,908 SALT including CT income tax, $2,808 mortgage interest, $19,001 charity). It also added the refund to AGI and skipped the 401(k) deferral. It used 2025 brackets and left out the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It used the married-filing-separately status and brackets for a single filer. It also counted the $6,147 tax-exempt pension as taxable and took charity on top of the standard deduction. Its final $48,557 does not match its own $45,287 − $750." us,scenario_120,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -9818,53 +9944,54 @@ us,scenario_120,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out us,scenario_120,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,snap,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It never quantified the Social Security modification (taxable SS minus 25% of benefits = $25,140 subtraction) and added back $2,852 of tax-exempt interest, which is not a CT addition here. Its asserted ~$205,000 base does not reproduce $12,130 under the schedule; the correct $207,524 CT AGI gives $11,239 plus $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It applied the $25,140 Social Security subtraction correctly but did not subtract the $1,750 state tax refund, which overstated CT AGI at $209,274 instead of $207,524. It also included only the $250 phase-out add-back and $180 middle-tax recapture, leaving out the $250 low-tax recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented a deduction for federal income tax paid, federal itemized deductions and an over-65 deduction to bring CT tax to zero. Connecticut allows none of these: its tax is computed on CT AGI of $207,524 with no itemized or age deduction, which yields $11,919." -us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a bare $7,000 guess with no computation. The correct path is federal AGI $234,414 minus the $25,140 Social Security subtraction and the $1,750 refund, giving $207,524. Graduated tax on that is $11,239, plus $680 of add-back and recapture. The $7,000 figure is consistent with treating most income as exempt and leaving out the upper brackets." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It wrongly asserted that no Social Security subtraction applies above $75,000. In fact Connecticut caps taxable benefits at 25% for such filers, which is a $25,140 subtraction. It also skipped the refund subtraction and added tax-exempt interest, reaching $236,883. It then cut its $13,247 figure to $11,890 with no identified credit." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the pre-2024 3%/5% bottom brackets and capped the schedule at 6% even though income exceeds $200,000, where the 6.5% bracket applies. Its $9,970 is also below what its own brackets give on $200,000 ($11,050), and it never added the $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It cut AGI by roughly the full $35,600 of taxable Social Security instead of the $25,140 excess over 25% of benefits, reaching $198,800. It then applied a 2%–5.5% schedule without the 6% and 6.5% brackets, which understated tax well below the $11,239 base plus $680 of recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Its CT AGI (~$207,495) and graduated tax (~$11,237) were essentially correct. However, it added only $250 + $180 of recapture and omitted the separate $250 low-tax recapture, so the total add-back is $680, not $430." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded that Connecticut gives no Social Security relief above $100,000 AGI, so it left out the $25,140 subtraction (taxable SS above 25% of benefits). It also skipped the $1,750 refund subtraction and taxed $234,414. It then omitted the $250 phase-out add-back and $430 of benefit recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It treated Social Security as largely exempt and subtracted Roth contributions. It also applied a personal exemption and credit that are fully phased out at this income. The result is $7,200, far below the graduated tax of $11,239 on $207,524 CT AGI, plus $680 of recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Its CT AGI of $209,891 is $2,367 too high because it did not subtract the $1,750 state tax refund. It also added only $180 of recapture, leaving out the $250 phase-out add-back and the $250 low-tax recapture." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable Social Security instead of the $25,140 excess over 25% of benefits. It also took a $15,000 personal exemption that is fully phased out for single filers at this AGI. Finally, it left out the $680 of phase-out add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It fully exempted Social Security instead of taxing 25% of benefits. It also applied a $1,549 personal tax credit, which is zero for a single filer at $207,524 CT AGI. The $680 of add-back and recapture was left out as well." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted only the state refund and left out the $25,140 Social Security subtraction. Connecticut caps taxable benefits at 25% for filers above $75,000 AGI, so CT AGI was overstated at $232,664 instead of $207,524." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable Social Security rather than $25,140, giving $197,049 instead of $207,524. It also used the old 3%/5% bottom brackets in place of 2%/4.5%. The $680 of phase-out add-back and benefit recapture was omitted." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It inverted the Social Security modification by subtracting only 25% of taxable SS ($8,904). The rule actually subtracts everything above 25% of benefits, which is $25,140. This overstated CT AGI at $223,731." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave a $7,800 figure with no computation. The correct graduated tax on $207,524 CT AGI is $11,239, plus $680 of add-back and recapture. A result this low is consistent with exempting all Social Security and pension income and leaving out the upper brackets and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It explicitly denied any Social Security exemption at high income. Connecticut in fact subtracts taxable benefits above 25% of benefits ($25,140) at every income level. Taxing near-full federal AGI overstated the result." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It understated federal AGI at $225,235 instead of $234,414, over-subtracted Social Security ($26,711 instead of $25,140), and skipped the refund subtraction, ending at $198,524 CT AGI. It also added only $150 of the $680 add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no computation, and its $11,770 is $149 short of the correct $11,239 base tax plus $680 of add-back and recapture on $207,524 CT AGI. That gap is consistent with leaving out part of the benefit recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It subtracted only $22,094 from federal AGI instead of the $25,140 Social Security subtraction plus the $1,750 refund ($26,890 total). This overstated CT taxable income at $212,320. It also added $660 of recapture instead of $680." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation, and its $13,125 overshoots by $1,206. That is consistent with taxing CT AGI well above $207,524 by skipping most of the $25,140 Social Security subtraction (taxable SS above 25% of benefits)." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It gave no computation, and its $9,706.75 falls $2,212 short of the correct $11,919. That is consistent with exempting all taxable Social Security instead of only the amount above 25% of benefits, and leaving out the $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It removed Social Security entirely and took a $15,000 'standard deduction' that Connecticut does not have. It also claimed a $200 property tax credit, which is fully phased out at this AGI, and left out the 6.5% bracket and $680 of recapture." -us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140. It then applied a flat 4.5% rate to the whole $197,049 rather than the graduated 2%–6.5% schedule, and left out the $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave $4,414 with no computation. The correct graduated tax on $207,524 CT AGI is $11,239, plus $680 of add-back and recapture. A result this low is consistent with treating most income as exempt or deductible." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced income by itemized deductions, which Connecticut does not allow because it taxes CT AGI directly. This understated tax well below the $11,239 graduated amount on $207,524 plus $680 of recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It subtracted 75% of federally taxable Social Security ($26,711) rather than taxable SS minus 25% of benefits ($25,140). It also added back $2,852 of tax-exempt interest that is not a CT addition here, which overstated taxable income at $208,805 instead of $207,524. Its $680 of add-back and recapture was correct." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed about $235,031 with no Social Security subtraction and no refund subtraction. It also applied about $1,240 of recapture instead of $680. The correct CT AGI after the $25,140 SS and $1,750 refund subtractions is $207,524." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It applied the schedule to about $234,000 and left out the $25,140 Social Security subtraction (taxable SS above 25% of benefits) and the $1,750 refund subtraction. That put CT AGI about $26,500 too high." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It denied Social Security relief and taxed the full $234,414, ignoring the $25,140 SS subtraction and the refund subtraction. Its $10,895 is also inconsistent with its own base, since the graduated schedule on $234,414 gives about $12,987 before recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It cut CT taxable income to $200,095, about $7,429 below the correct $207,524, by over-subtracting beyond the $25,140 Social Security and $1,750 refund modifications. It also applied $590 of add-back and recapture instead of $680." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed the full $234,414 federal AGI with no Social Security or refund subtraction. It also inflated recapture far beyond $680. The correct CT AGI after the $26,890 of subtractions is $207,524." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted only the state refund, reaching $232,664, and left out the $25,140 Social Security subtraction that limits taxable benefits to 25% for filers above $75,000 AGI." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It reduced CT taxable income to $200,095, about $7,429 below the correct $207,524, by subtracting more than the $25,140 Social Security and $1,750 refund modifications. It also computed only $90 of recapture where the correct middle-tax recapture is $180 and the low-tax recapture is $250, a total of $590 against the correct $680." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave $7,850 with no computation. The correct graduated tax on $207,524 CT AGI is $11,239, plus $680 of add-back and recapture. The low figure is consistent with exempting all Social Security and pension income and leaving out the top brackets and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140, the amount above 25% of benefits. That put CT AGI at $197,049 instead of $207,524. It also left out the $250 phase-out add-back and $430 of benefit recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It subtracted all taxable Social Security and applied a 100% pension exemption, which Connecticut phases out entirely for single filers at this AGI. It also used the old 3%/5% brackets and added only $200 of recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added tax-exempt interest and left out both the $25,140 Social Security subtraction and the $1,750 refund subtraction. It then used the old 3%/5% bracket base of $11,050 instead of $10,750 and omitted the $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable Social Security and added tax-exempt interest, skipping the refund subtraction. It then applied the outdated 3%/5% bottom brackets and left out the $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It taxed $234,385 with no Social Security or refund subtraction and left out the $680 of add-back and recapture. It also subtracted a $300 property tax credit that is fully phased out at this AGI." -us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score." -us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 with no reasoning, which implies no Connecticut liability at all. The correct path is CT AGI of $207,524 after the $25,140 Social Security and $1,750 refund subtractions, taxed at $11,239 plus $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It claimed the Social Security exclusion phases out entirely and invented a Connecticut charitable itemized deduction, which Connecticut does not have. It taxed $218,938 instead of $207,524 and left out the $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It excluded all Social Security, subtracted tax-exempt pension income that was never in AGI, and took a $15,000 personal exemption that is phased out at this income. It also used old 3%/5% brackets. Together these understated CT taxable income at about $136,825 instead of $207,524." -us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It applied the married-filing-separately schedule instead of the single schedule. It also subtracted a $500 property tax credit, which exceeds the $300 maximum and is fully phased out at this AGI." +us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It added back $2,852 of tax-exempt interest and never worked out the SS subtraction, only saying the subtraction was 'limited'. The correct subtraction is $25,140, so that only 25% of benefits stays taxable. It also never itemized the $250 phase-out add-back or the $430 recapture, so its ~$205,000 base and $12,130 tax don't follow from the CT schedule." +us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It got the $25,140 SS subtraction right but left out the $1,750 SALT refund subtraction, giving a CT AGI of $209,274 instead of $207,524. It added only the $250 phase-out and $180 middle-tax recapture, missing the separate $250 low-tax recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It invented a CT deduction for federal income tax paid, an over-65 deduction and federal-style itemizing. Connecticut has none of these, so CT AGI is $207,524 and is fully taxable, which gives $11,919, not $0." +us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It gave a $7,000 estimate with no calculation. Applying the CT schedule to $207,524 of CT AGI (federal AGI less the $25,140 SS subtraction and $1,750 refund) gives $11,239, plus $680 of phase-out and recapture. That is $11,919." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It wrongly concluded that no SS subtraction applies above $75k, when CT still subtracts $25,140 so that only 25% of benefits is taxed. It also skipped the $1,750 refund subtraction, added back tax-exempt interest and used a 3% bottom rate. Its final $11,890 is an unexplained cut from its own $13,247 subtotal." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It used ~$200k of taxable income instead of $207,524, stopped the schedule at 6% with the old 3%/5% lower rates, and left out the 6.5% bracket and the $680 of phase-out and recapture. Its $9,970 is below even the $11,050 its own rates give at $200k." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It applied only 2%–5.5% rates to ~$198,800 of taxable income, leaving out the 6% bracket ($100k–$200k) and the 6.5% bracket above $200k. It also understated CT AGI, which is $207,524 after the $25,140 SS and $1,750 refund subtractions." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Its CT AGI (~$207,495) and graduated tax (~$11,237) are essentially right, but it added only the $250 phase-out and $180 middle-tax recapture. It left out the separate $250 low-tax recapture, so it came in $250 short." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It claimed CT gives no SS exemption above $100k AGI, but for high-AGI single filers CT still subtracts taxable SS above 25% of benefits ($25,140). It also skipped the $1,750 SALT refund subtraction, taxed $234,414 instead of $207,524, and left out the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It estimated CT AGI at ~$198,000 by subtracting Roth and IRA contributions and a vague SS exemption. Its $7,200 also contradicts the graduated schedule, which gives over $10,700 at that income. The correct base is $207,524 after the $25,140 SS and $1,750 refund subtractions." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It skipped the $1,750 SALT refund subtraction, giving a CT AGI of ~$209,891 instead of $207,524. It added only the $180 middle-tax recapture and missed both the $250 phase-out of the 2% bracket and the $250 low-tax recapture." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable SS instead of $25,140, the amount that leaves 25% of benefits taxable. It also added tax-exempt interest, skipped the refund subtraction, took a $15,000 personal exemption that is fully phased out at this income, and left out the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It exempted Social Security in full instead of subtracting only $25,140. It also took a $1,549 personal tax credit, which is fully phased out at a CT AGI above $200k." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted only the $1,750 SALT refund and missed the $25,140 Social Security subtraction, which limits taxable benefits to 25% of benefits for high-AGI single filers. As a result it taxed $232,664 instead of $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable SS instead of $25,140, so 25% of benefits stayed out of the base. It also used the repealed 3%/5% lower rates instead of 2%/4.5% and left out the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It got the 25% rule backwards, subtracting only 25% of taxable SS ($8,904). The correct subtraction is taxable SS minus 25% of benefits ($25,140), and the error pushed CT AGI to $223,731 instead of $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave $7,800 with no calculation. The correct path is CT AGI of $207,524 after the $25,140 SS and $1,750 refund subtractions, $11,239 of graduated tax and $680 of phase-out and recapture, for $11,919." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It explicitly applied no Social Security subtraction because of the high income. CT still subtracts $25,140 at this AGI so that only 25% of benefits is taxed, so it taxed roughly $234k instead of $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It understated federal AGI ($225,235 vs $234,414) and used a $26,711 SS subtraction (75% of taxable SS) instead of $25,140, with no refund subtraction. It also added only ~$150 of recapture instead of the full $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no calculation. CT AGI of $207,523.97 gives $11,239.06 of graduated tax plus $680 of phase-out and recapture, and its $11,770 is $149 short, which points to an incomplete add-back." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It subtracted only ~$22,094 in total from federal AGI, reaching $212,320 instead of the $26,890 subtraction ($25,140 SS plus $1,750 refund) that gives $207,524. The extra ~$4,800 in the base, taxed at 6.5%, accounts for the overstatement." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no calculation. $13,125 matches taxing close to full federal AGI without the $25,140 Social Security subtraction and $1,750 refund subtraction that bring CT AGI down to $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It gave no calculation. $9,706.75 matches over-subtracting Social Security (excluding all of it) and/or dropping the 6.5% bracket and the $680 of phase-out and recapture that apply at a CT AGI of $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It treated Social Security as fully exempt instead of subtracting only $25,140. It also took a $15,000 'standard deduction' that CT doesn't have and a $200 property tax credit that is phased out at this income, and stopped the rates at 6% with no recapture." +us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable SS instead of $25,140 and taxed the whole base at a flat 4.5%. It never applied the graduated 2%–6.5% schedule or the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave $4,414 with no calculation. That is far below the $11,919 from taxing $207,524 of CT AGI on the graduated schedule plus $680 of phase-out and recapture, which suggests it took deductions CT doesn't allow or left out most of the income." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced CT taxable income by itemized deductions, but Connecticut allows no itemized deductions and taxes CT AGI directly. The correct base is $207,524, after only the $25,140 SS and $1,750 refund subtractions." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It subtracted 75% of taxable SS ($26,711) instead of taxable SS minus 25% of benefits ($25,140), and it added back $2,852 of tax-exempt interest that isn't part of the CT base here. That raised CT AGI to $208,805 and overstated the tax by about $83, although its $680 add-back total is correct." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed ~$235,031 with no Social Security subtraction and no SALT refund subtraction. The $25,140 SS subtraction still applies at this AGI, so the correct base is $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It applied the schedule to ~$234,000 of CT AGI, skipping the $25,140 Social Security subtraction (taxable SS above 25% of benefits) and the $1,750 refund subtraction that bring the base to $207,524." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It said income was too high for any Social Security relief and used $234,414, missing the $25,140 subtraction and the $1,750 refund subtraction. Its $10,895 also contradicts the CT schedule on its own base, which gives about $13,000 before recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It over-subtracted Social Security, about $32,569 implied instead of $25,140, and reached a $200,095 base instead of $207,524. It also added only $590 of phase-out and recapture instead of $680." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed the full $234,414 federal AGI with no $25,140 Social Security subtraction or $1,750 refund subtraction, and it overstated the rate and recapture amounts well beyond the $680 cap." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted only the SALT refund and missed the $25,140 Social Security subtraction. Its $13,286.91 also reflects the repealed 3%/5% lower rates applied to $234,414, with no phase-out or recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It over-subtracted Social Security (about $32,569 implied vs $25,140), giving taxable income of $200,095 instead of $207,524. It added only $90 of recapture, when the correct add-backs are the $250 phase-out plus $250 low-tax and $180 middle-tax recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It gave $7,850 with no calculation, far below the $11,919 from taxing $207,524 of CT AGI at 2%–6.5% plus $680 of phase-out and recapture. That points to an over-stated exemption or deduction or to dropped upper brackets." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable SS instead of $25,140, which kept CT AGI under $200k at $197,049 and avoided the 6.5% bracket. It also left out the $250 phase-out add-back and the $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It applied a 100% pension exemption that is fully phased out for single filers at this AGI. It also subtracted all taxable SS instead of $25,140 and used the old 3%/5% lower rates." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added $2,852 of tax-exempt interest and took no Social Security or SALT refund subtraction, reaching $235,516 instead of $207,524. Its $11,050 base at $200k also reflects the repealed 3%/5% lower rates." +us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It subtracted all $35,615 of taxable SS instead of $25,140, added tax-exempt interest and skipped the refund subtraction. It then applied the old 3%/5% lower rates and left out the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It taxed $234,385 with no Social Security or SALT refund subtraction. It also took a $300 property tax credit that is fully phased out at this AGI and left out the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It answered $0 with no reasoning, ignoring $207,524 of CT taxable income. Under the graduated 2%–6.5% schedule plus $680 of phase-out and recapture, that income produces $11,919 of tax." +us,scenario_120,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Its ~$208,000 base and graduated tax are close, but it treated the high-income adjustments only as phased-out benefits. It never added the $250 phase-out of the 2% bracket or the $430 low-tax and middle-tax recapture." +us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It invented a CT itemized charitable deduction ($15,476), but Connecticut has no itemized deductions. It also skipped the $25,140 SS and $1,750 refund subtractions and left out the $680 of phase-out and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It subtracted the full $41,900 of SS and the $6,147 tax-exempt pension from an AGI that never included them, which double-excludes both. It also took a $15,000 personal exemption that is fully phased out and used old 3%/5% rates." +us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It understated taxable income ($202,564 vs $207,524) and subtracted a $500 property tax credit. That credit is capped at $300 and fully phased out at this AGI, so there are no nonrefundable credits to subtract." us,scenario_120,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -9886,22 +10013,23 @@ us,scenario_121,head_medicaid_eligible,grok-4.6,llm_error,categorical_eligibilit us,scenario_121,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied a medically needy spend-down plus an asset limit to reach eligibility; neither is a Medicaid pathway in the engine, where medical expenses and the $630 bank balance feed no category determination. With SSI at $0 and MAGI at 1.61 x FPL, medicaid_category resolves to NONE." us,scenario_121,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for head_medicaid_eligible, so the key was absent from the submitted outputs rather than answered incorrectly. This is a failure to produce the required output object, not a substantive error about SC Medicaid pathways." us,scenario_121,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model equated the `is_disabled` flag and $13,044 of disability benefits with SSI-related Medicaid, but that pathway keys on actual SSI receipt, which is $0 because $25,665 of wages plus $13,044 of unearned disability benefits blow past the SSI federal benefit rate after the standard disregards. Meeting disability criteria establishes no Medicaid category on its own in a non-expansion state, and MAGI at 1.61 x FPL clears every SC MAGI limit." -us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"Relabeled the generic $13,044 disability benefits as Social Security disability benefits and said they alone qualify someone for Medicare before 65. It skipped the 24-month SSDI entitlement requirement. SSDI months are unlisted and therefore 0, and ESRD is false, so a 53-year-old fails every Medicare pathway." -us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"Treated the head as an SSDI recipient even though the input is only generic disability benefits. It also treated SSDI receipt itself as Medicare eligibility and never applied the 24-month entitlement requirement, which is unmet because no SSDI duration is given." -us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Named the 24-month SSDI waiting period but assumed it was met because disability status is constant within the year. Constant status during the year does not show 24 prior months of SSDI entitlement. SSDI months are unlisted and therefore 0, and the $13,044 is generic disability benefits, not SSDI." -us,scenario_121,head_medicare_eligible,deepseek-v4-flash-0731,llm_error,age_disability,False,"Called the generic disability benefits SSDI and asserted eligibility 'after the qualifying period'. Nothing in the facts supports the 24-month SSDI entitlement; it is 0 by the unlisted-input rule, so the head, at 53, fails the Medicare age and disability tests." -us,scenario_121,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"Explicitly assumed the 24-month SSDI waiting period had passed, which the prompt forbids by making unlisted inputs 0 and barring inferred benefit receipt. It also treated generic disability benefits as SSDI, when SSDI entitlement is the only non-age pathway that applies here." -us,scenario_121,head_medicare_eligible,deepseek-v4-pro-0813,llm_error,age_disability,False,"Treated being disabled and receiving generic disability benefits as meeting Medicare's disability test. That test requires 24 months of SSDI entitlement (or ESRD or ALS), and none of these is present. The head is 53, below the age-65 threshold." -us,scenario_121,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Asserted that the head 'receives SSDI' when the input is only generic disability benefits. It then equated SSDI receipt with Medicare eligibility, skipping the 24-month entitlement requirement, which is 0 months here." -us,scenario_121,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"Labeled the generic disability benefits as SSDI and applied the two-year rule as if it were already satisfied. No SSDI duration is given, so it is 0, the disability pathway fails, and the head is under 65." -us,scenario_121,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"Invented the fact that the head 'has met the waiting period requirements'. The household facts contain no SSDI entitlement or duration, so the 24-month SSDI requirement is unmet, and at 53 the head fails the age test." -us,scenario_121,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"Recast the $13,044 generic disability benefits as Social Security disability benefits and treated them as generally conferring Medicare. It ignored the 24-month SSDI entitlement requirement, which is unmet with 0 listed months." -us,scenario_121,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"Equated the generic disability benefits with SSDI and treated disability status as enough for Medicare. It never applied the 24-month SSDI entitlement requirement, so a 53-year-old with no ESRD is ineligible." -us,scenario_121,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"Treated the is-disabled flag alone as qualifying for Medicare. The under-65 disability pathway requires 24 months of SSDI entitlement (or ESRD or ALS), not a disability status, and none of these is present." -us,scenario_121,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"Applied a 'disability pathway' that it keyed to being disabled and receiving generic disability benefits. The actual pathway requires 24 months of SSDI entitlement, and that is 0 because SSDI receipt and duration are unlisted." -us,scenario_121,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"Treated disability status plus generic disability benefits as satisfying Medicare's disability test. It omitted the required 24 months of SSDI entitlement, which is 0 here, so the 53-year-old head fails the age-65 threshold with no alternative pathway." -us,scenario_121,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"Invoked a 'disability-based Medicare eligibility rule' as if disability status were enough. That rule requires 24 months of SSDI entitlement (or ESRD or ALS), none of which is in the household facts." -us,scenario_121,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"Inferred both SSDI receipt ('consistent with SSDI') and completion of the statutory waiting period from generic disability benefits. The prompt forbids inferring unlisted benefit receipt and sets unlisted inputs to 0, so the 24-month SSDI requirement is unmet and the head, at 53, is ineligible." +us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model assumed the generic $13,044 'disability benefits' was Social Security disability. It then treated that as Medicare entitlement under 65 without the 24 months of SSDI receipt the rule requires, which is 0 here because it was not listed. The head is 53 with no qualifying SSDI duration and no ESRD, so they are not Medicare-eligible." +us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model equated the disabled flag and generic disability benefits with SSDI and concluded that SSDI recipients qualify for Medicare. That ignores the requirement of at least 24 months of SSDI receipt. No SSDI months were given, so the count is 0, and the 53-year-old head fails both the age-65 test and the disability pathway." +us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the 24-month SSDI entitlement rule but then assumed the waiting period was met because disability status is constant. 'Constant throughout the year' does not supply 24 months of prior SSDI receipt, and the unlisted count of SSDI months is 0. The generic disability benefits were also not Social Security disability, so the head is not eligible." +us,scenario_121,head_medicare_eligible,deepseek-v4-flash-0731,llm_error,age_disability,False,"The model relabeled the generic disability benefits as SSDI and assumed the qualifying period had passed. No SSDI receipt or months of receipt were given, so the count defaults to 0 and the 24-month requirement is not met. At age 53 with no ESRD, the head is not Medicare-eligible." +us,scenario_121,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model explicitly assumed that the 24-month SSDI waiting period had passed, which breaks the instruction to treat unlisted numeric inputs as 0 and not to infer benefit receipt. With 0 months of SSDI receipt and only a generic disability-benefits amount, the 53-year-old head has no under-65 Medicare pathway." +us,scenario_121,head_medicare_eligible,deepseek-v4-pro-0813,llm_error,age_disability,False,"The model treated 'under 65, disabled, and receiving disability benefits' as meeting the Medicare disability test. The test actually requires at least 24 months of Social Security disability receipt (or ESRD), and none was given. The head is therefore not eligible." +us,scenario_121,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model asserted that the head receives SSDI, but the input is generic disability benefits, not Social Security disability. It also skipped the 24-month SSDI receipt requirement, which is 0 months here, so the 53-year-old head is not Medicare-eligible." +us,scenario_121,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model recognized the two-year waiting period but assumed it was satisfied without any input showing 24 months of SSDI receipt. That count defaults to 0. The generic disability-benefits amount is not SSDI, so the head fails both the age and disability pathways." +us,scenario_121,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model stated that the head 'has met the waiting period requirements', but no months of SSDI receipt were provided, so the count is 0. Being disabled with generic disability income does not satisfy the 24-month SSDI test, and the head is 53." +us,scenario_121,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model read the generic disability benefits as Social Security disability benefits and treated receipt as sufficient for Medicare. Under-65 eligibility requires at least 24 months of SSDI receipt (or ESRD), and that count is 0 here, so the head is not eligible." +us,scenario_121,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model relabeled the generic disability benefits as SSDI and granted Medicare from the disabled flag alone. It never applied the requirement of at least 24 months of SSDI receipt, which is 0 here. The 53-year-old head is not Medicare-eligible." +us,scenario_121,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model treated the 'is disabled' flag by itself as qualifying for Medicare. Disability status alone does not confer Medicare: the head needs at least 24 months of SSDI receipt or ESRD, and both are absent (0) for this 53-year-old." +us,scenario_121,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model invoked the under-65 disability pathway using the disabled flag and generic disability benefits. That pathway requires at least 24 months of Social Security disability receipt, and the unlisted count is 0, so the head does not qualify." +us,scenario_121,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"The model concluded that disability status plus generic disability benefits qualify the head for Medicare. It skipped the requirement of at least 24 months of SSDI receipt, which is 0 here, and the generic benefits are not SSDI. The head, at 53, is not eligible." +us,scenario_121,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied the disability-based Medicare rule without checking its trigger: at least 24 months of SSDI receipt or ESRD. Neither is present (the SSDI-months count is 0), so the 53-year-old head is not Medicare-eligible." +us,scenario_121,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model judged the generic disability benefits to be 'consistent with SSDI' and assumed the statutory waiting period was met. Both assumptions infer facts that were not given: the SSDI-months count is 0, so the 24-month test fails and the head is not eligible." +us,scenario_121,head_medicare_eligible,muse-wonder,llm_error,age_disability,False,"The model granted Medicare through the disability pathway because the head is disabled with substantial disability benefits. Benefit size is irrelevant: what matters is at least 24 months of SSDI receipt or ESRD, and neither is present. The 53-year-old head is not eligible." us,scenario_121,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_121,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the $8,389 employer-sponsored insurance premium as a pre-tax payroll deduction even though no pre-tax treatment was listed. FICA applies to the full $25,665 of wages, not $17,276." us,scenario_121,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from FICA wages. With no listed pre-tax deduction, Social Security and Medicare taxes apply to the full $25,665." @@ -9911,36 +10039,37 @@ us,scenario_121,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"T us,scenario_121,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model wrongly treated employer-sponsored insurance premiums as automatically reducing the FICA wage base. The full $25,665 is subject to employee Social Security and Medicare tax." us,scenario_121,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct full-wage FICA calculation but submitted $1,911, which does not follow from its own formula. Applying 6.2% and 1.45% to $25,665 produces $1,591.23 and $372.14, totaling $1,963.37; South Carolina adds no employee payroll tax here." us,scenario_121,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,snap,claude-fable-5,llm_error,age_disability,False,"The model waived the gross income test because the head is disabled, then subtracted $2,681.67 a month in excess medical costs to get about $11 of net income and a $3,540 benefit. SNAP counts a member as disabled only if they receive SSI, Social Security disability or similar government disability payments (7 CFR 271.2); generic disability benefits do not qualify. The household therefore faces the gross test, and $3,325.75 a month exceeds South Carolina's 130% FPL limit (about $1,696), so SNAP is $0." -us,scenario_121,snap,claude-fable-5.1,llm_error,age_disability,False,"The model declared 'no gross income test' for a disabled one-person household and used about $3,381 of excess medical costs to zero out net income, giving the $3,576 maximum. The head receives no SSI or Social Security disability, so they are not a SNAP disabled member and neither the gross-test exemption nor the medical deduction applies. Gross income of about $3,326 a month fails South Carolina's 130% FPL test, so the benefit is $0." -us,scenario_121,snap,claude-opus-4.7,llm_error,age_disability,False,"The model saw that $3,326 a month exceeds the broad-based categorical eligibility limit, which it took as 200% FPL (South Carolina's is actually 130%). It then exempted the household anyway as disabled, and trimmed its $3,504 maximum to $3,168 with no stated basis. The head does not meet SNAP's receipt-based disability definition (no SSI or Social Security disability), so failing the gross test gives $0." -us,scenario_121,snap,claude-opus-4.8,llm_error,age_disability,False,"The model applied only the net income test because the head is disabled, left $1,200 of child support out of gross income, and answered $4,860, which is more than the one-person maximum of about $3,576 a year. The generic disability benefits are not SSI or Social Security disability, so the head is not SNAP-disabled. The gross income test applies and gross income above 130% FPL makes the household ineligible, so SNAP is $0." -us,scenario_121,snap,claude-opus-5.5,llm_error,age_disability,False,"The model put the household on the disabled track: net income test only, the $4,500 asset limit, and a $2,682 monthly medical deduction, giving about $7 of net income and $3,552 a year. SNAP's disabled status depends on receiving SSI, Social Security disability or similar payments, and this head receives none of those. The household therefore fails the 130% FPL gross test with about $3,326 a month and gets $0." -us,scenario_121,snap,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly found that $39,909 exceeds the 130% FPL gross limit, then overrode that result by exempting the household as disabled and deducting $32,180 of medical costs to reach $3,432. The head receives no SSI or Social Security disability, so they are not a SNAP disabled member and the failed gross test is final, making SNAP $0." -us,scenario_121,snap,claude-sonnet-5,llm_error,age_disability,False,"The model admitted gross income exceeds the one-person limit but applied the elderly/disabled exemption. It then built deductions from employer premiums, the $2,200 premium counted twice, and shelter costs invented from a mortgage balance with no stated payment, and cut the $3,504 maximum to an arbitrary $3,200. The head does not meet SNAP's receipt-based disability definition, so the gross test controls and SNAP is $0." -us,scenario_121,snap,claude-sonnet-5.5,llm_error,age_disability,False,"The model treated the head as a SNAP disabled member and used about $32,600 of medical costs to bring net income to zero and claim the $298-a-month maximum ($3,576 a year). The head receives no SSI or Social Security disability, so the household gets no gross-test exemption and no medical deduction. Gross income of about $3,326 a month exceeds South Carolina's 130% FPL limit, so SNAP is $0." -us,scenario_121,snap,deepseek-v4-flash-0731,llm_error,age_disability,False,"The model called the head's household 'disabled' and applied the uncapped medical deduction to reach near-zero net income and a $299-a-month maximum ($3,588). Under 7 CFR 271.2, SNAP disability requires receiving SSI, Social Security disability or similar payments, which this head does not. The household therefore fails the 130% FPL gross test and receives $0." -us,scenario_121,snap,deepseek-v4-pro-0813,llm_error,age_disability,False,"The model exempted the household from the gross income test as disabled and used the excess medical deduction to reach zero net income and a $292-a-month maximum ($3,504). The head's generic disability benefits are not SSI or Social Security disability, so the household is non-elderly and non-disabled for SNAP. Its gross income of about $3,326 a month is over the 130% FPL limit, so SNAP is $0." -us,scenario_121,snap,deepseek-v4.1-flash,llm_error,age_disability,False,"The model called the household 'exempt from gross income limit' because it is disabled and computed about $294 a month ($3,520.20 a year) after a medical deduction. SNAP's disabled status is receipt-based (SSI, Social Security disability or similar), and this head qualifies under none of those. The gross test applies, gross income exceeds 130% FPL, and SNAP is $0." -us,scenario_121,snap,gemini-3-flash-preview,llm_error,age_disability,False,"The model waived the gross test because the head is disabled, took a medical deduction of more than $2,800, added a shelter deduction even though no housing costs are listed, and awarded the $292 maximum ($3,504). The head receives no SSI or Social Security disability, so the household must pass the gross test. It fails with about $3,326 a month against South Carolina's roughly $1,696 limit, so SNAP is $0." -us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model exempted the household from the gross income test because the head is disabled and used medical deductions to reach zero net income and a $291-a-month maximum ($3,492). SNAP's disabled-member definition requires receiving SSI, Social Security disability or similar government payments, which the head does not. The household fails the 130% FPL gross test and receives $0." -us,scenario_121,snap,gemini-3.5-flash,llm_error,age_disability,False,"The model said the gross income limit is waived because the head is disabled and let medical costs reduce net income to zero, giving an estimated $3,504 maximum. The head's generic disability benefits are not SSI or Social Security disability, so the head is not a SNAP disabled member. Gross income of about $3,326 a month fails South Carolina's 130% FPL test, so SNAP is $0." -us,scenario_121,snap,gemini-3.6-flash,llm_error,age_disability,False,"The model exempted the head from the gross test as disabled and subtracted a $2,865 excess medical deduction to reach zero net income and $3,504 a year. The head receives no SSI or Social Security disability, so neither the gross-test exemption nor the medical deduction applies. Monthly gross income of about $3,326 is above the roughly $1,696 limit, so SNAP is $0." -us,scenario_121,snap,gemini-3.7-flash,llm_error,age_disability,False,"The model exempted the head from the gross income test as disabled and let the excess medical deduction cancel almost all net income, giving $3,336 a year. SNAP counts a member as disabled only if they receive SSI, Social Security disability or similar payments, and this head does not. The household fails the 130% FPL gross test and receives $0." -us,scenario_121,snap,glm-5.3,llm_error,age_disability,False,"The model treated the household as disabled, skipped the gross test, left out the $1,200 of child support (using $38,709), and used more than $30,000 of medical costs to reach zero net income and $3,504. The head does not meet SNAP's receipt-based disability definition, so the gross test applies. The full $39,909 of gross income exceeds South Carolina's 130% FPL limit, so SNAP is $0." -us,scenario_121,snap,gpt-5.5,llm_error,age_disability,False,"The model treated this as a disabled one-person SNAP household, so it allowed the medical-expense deduction and awarded the $298-a-month maximum ($3,576). The head receives no SSI or Social Security disability, so they are not SNAP-disabled and the household must pass the gross test. It fails with about $3,326 a month against a limit of about $1,696, so SNAP is $0." -us,scenario_121,snap,gpt-5.6-sol,llm_error,age_disability,False,"The model let the disabled one-person household deduct its large medical costs, reaching zero net income and the $298-a-month maximum ($3,576). SNAP disability status requires receiving SSI, Social Security disability or similar government payments, which this head lacks. The household therefore fails the 130% FPL gross income test and receives $0." -us,scenario_121,snap,gpt-5.6-terra,llm_error,age_disability,False,"The model treated the household as SNAP-disabled and let medical costs reduce net income to zero, giving the $298-a-month maximum ($3,576). The head's generic disability benefits are not SSI or Social Security disability, so the household gets no gross-test exemption or medical deduction. Gross income of about $3,326 a month exceeds South Carolina's 130% FPL limit, so SNAP is $0." -us,scenario_121,snap,gpt-6-astra,llm_error,age_disability,False,"The model subtracted a $32,180 'disability-related medical deduction' to reach $88 of annual net income and a $3,549.60 benefit, which assumes the head counts as disabled for SNAP. Under 7 CFR 271.2 that status requires receiving SSI, Social Security disability or similar payments, which this head does not. The gross test therefore applies and $39,909 of gross income makes the household ineligible, so SNAP is $0." -us,scenario_121,snap,gpt-6-luna,llm_error,age_disability,False,"The model applied the elderly/disabled medical deduction to bring net income to zero and awarded the $298-a-month maximum ($3,576). The head receives no SSI or Social Security disability, so the household is not elderly/disabled for SNAP and must pass the gross test. Its gross income of about $3,326 a month is far above the roughly $1,696 limit, so SNAP is $0." -us,scenario_121,snap,gpt-6-sol,llm_error,age_disability,False,"The model treated the head as a disabled SNAP member, deducted the medical costs to reach zero net income, and awarded $298 a month for 12 months ($3,576). SNAP's disability definition is based on receiving SSI, Social Security disability or similar payments, and the head's generic disability benefits do not qualify. The household fails the 130% FPL gross test and receives $0." -us,scenario_121,snap,gpt-6.1-sol,llm_error,age_disability,False,"The model stated that 'disability permits the net-income eligibility test', skipped the gross test, and took a $32,180 medical deduction to reach $88 of net income and $3,549.60. The head receives no SSI or Social Security disability, so the household is not SNAP-disabled and must pass the gross test. It fails with $39,909 against South Carolina's 130% FPL limit of about $20,345 a year, so SNAP is $0." -us,scenario_121,snap,grok-4.6,llm_error,age_disability,False,"The model called this a 'disabled SNAP unit' that skips the gross income test and used the excess medical deduction to reach zero net income and the $292 maximum ($3,504). The head does not meet SNAP's receipt-based disability definition (no SSI or Social Security disability). Gross income of about $3,326 a month therefore fails the 130% FPL test, so SNAP is $0." -us,scenario_121,snap,grok-4.7,llm_error,age_disability,False,"The model waived the gross test for a 'disabled one-person SNAP unit' and subtracted a $31,780 medical deduction to reach $488 of net income and $3,429.60. The head's generic disability benefits are not SSI or Social Security disability, so the household is not SNAP-disabled. The $39,909 of gross income exceeds the 130% FPL limit and makes the household ineligible, so SNAP is $0." -us,scenario_121,snap,grok-build-0.1,llm_error,age_disability,False,"The model applied only the net income test because the head is disabled, let medical costs reduce net income to zero, and used an estimated maximum of $3,600 a year. The head receives no SSI or Social Security disability, so they are not a SNAP disabled member and the gross test applies. Gross income of about $3,326 a month fails South Carolina's 130% FPL limit, so SNAP is $0." -us,scenario_121,snap,inkling,llm_error,age_disability,False,"The model treated this as a disabled one-person household, deducted large out-of-pocket medical costs to reach zero net income, and awarded the $298 maximum ($3,576). SNAP's disabled status requires receiving SSI, Social Security disability or similar government payments, which this head lacks. The household must pass the gross test, fails it at about $3,326 a month, and receives $0." -us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value and no explanation, so there was no usable answer to compare. The correct result is $0: the head receives no SSI or Social Security disability, so the household must pass the 130% FPL gross income test, and about $3,326 a month of gross income fails it." -us,scenario_121,snap,kimi-k3,llm_error,age_disability,False,"The model gave the head the disabled-member earned-income and uncapped medical deductions, counting both $2,200 premium entries, to reach zero net income and the $298 maximum ($3,576). It never applied the gross test, even though the head receives no SSI or Social Security disability and so is not SNAP-disabled. Gross income of $39,909 (about $3,326 a month) exceeds South Carolina's 130% FPL limit, so SNAP is $0." +us,scenario_121,snap,claude-fable-5,llm_error,age_disability,False,"Waived the gross income test because the head is disabled. The head does not meet SNAP's disabled-member definition, which requires receiving SSI, SSDI, or similar benefits, so $3,326/mo of gross income fails SC's 200% FPL BBCE limit (about $2,608/mo). The $2,682/mo excess medical deduction it then used is unavailable, and the benefit is $0, not $3,540." +us,scenario_121,snap,claude-fable-5.1,llm_error,age_disability,False,"Asserted 'no gross income test' for a disabled one-person household. The head's generic disability benefits and above-SGA wages do not make them a SNAP elderly/disabled member, so the gross income test applies. Gross income of $3,326/mo exceeds the 200% FPL limit, giving $0 rather than the $298/mo maximum allotment." +us,scenario_121,snap,claude-opus-4.7,llm_error,age_disability,False,"Correctly noted SC's 200% FPL BBCE gross limit (about $2,510/mo) and that $3,326/mo exceeds it. It then exempted the household as disabled, but the head does not qualify as a SNAP disabled member (no SSI/SSDI receipt), so the gross test failure is final and SNAP is $0. Its arbitrary drop of the allotment to $264/mo does not change the eligibility error." +us,scenario_121,snap,claude-opus-4.8,llm_error,age_disability,False,"Applied only the net income test on the theory that the household is disabled. The head is not a SNAP elderly/disabled member, so the gross income test applies, and the full $39,909 (it also wrongly dropped the $1,200 child support) exceeds the 200% FPL limit. Its $4,860 also exceeds the annual one-person maximum allotment." +us,scenario_121,snap,claude-opus-5.5,llm_error,age_disability,False,"Used the elderly/disabled pathway: net income test only, $4,500 asset limit, and the excess medical deduction. The head does not meet SNAP's disabled-member definition, which requires receipt of SSI, SSDI, or similar benefits. The gross income test therefore applies, and $3,326/mo exceeds the 200% FPL limit, so SNAP is $0." +us,scenario_121,snap,claude-sonnet-4.6,llm_error,age_disability,False,"Correctly found that $39,909 fails the gross income test, then overrode that result with the disabled-household exemption. The head is not a SNAP disabled member because they receive only generic disability benefits, not SSI or SSDI, and their wages exceed SGA. The gross-test failure stands, the $32,180 medical deduction is unavailable, and SNAP is $0." +us,scenario_121,snap,claude-sonnet-5,llm_error,age_disability,False,"Applied the elderly/disabled net-income-only test even though the head is not a SNAP disabled member, so gross income of $39,909 fails the 200% FPL limit and SNAP is $0. It also wrongly counted the $8,389 ESI premium as a medical expense and invented shelter costs from a mortgage balance with no listed payment." +us,scenario_121,snap,claude-sonnet-5.5,llm_error,age_disability,False,"Treated the household as a SNAP disabled household eligible for the medical deduction and exempt from the gross test. The head's general disability flag does not satisfy SNAP's benefit-receipt definition of disability, so $3,326/mo of gross income fails the 200% FPL limit and the benefit is $0." +us,scenario_121,snap,deepseek-v4-flash-0731,llm_error,age_disability,False,"Assumed a disabled one-person household with an uncapped medical deduction. The head does not qualify as a SNAP elderly/disabled member, so the gross income test applies, and $39,909/yr exceeds the 200% FPL BBCE limit, giving $0." +us,scenario_121,snap,deepseek-v4-pro-0813,llm_error,age_disability,False,"Exempted the household from the gross income test as disabled. SNAP disability status requires receipt of SSI, SSDI, or similar benefits, which the head does not have, so gross income of $3,326/mo fails the roughly $2,608/mo 200% FPL limit and SNAP is $0." +us,scenario_121,snap,deepseek-v4.1-flash,llm_error,age_disability,False,"Waived the gross income limit for a disabled household and applied the medical deduction. The head is not a SNAP disabled member, so gross income of $39,909 exceeds the 200% FPL gross limit and the household is ineligible ($0)." +us,scenario_121,snap,gemini-3-flash-preview,llm_error,age_disability,False,"Waived the gross income test for a disabled member and applied the medical and shelter deductions. The head does not meet SNAP's disabled-member definition, so $3,325/mo fails the 200% FPL gross limit and SNAP is $0. No shelter cost was listed either." +us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,age_disability,False,"Claimed the disabled head exempts the household from the gross income test. The head receives no SSI or SSDI and is not SNAP-disabled, so the gross test applies, and $39,909/yr exceeds the 200% FPL limit. The result is $0, not the $291/mo maximum." +us,scenario_121,snap,gemini-3.5-flash,llm_error,age_disability,False,"Waived the gross income limit because the head is disabled. SNAP's disability definition requires receiving qualifying disability benefits, which the head does not, so gross income of $3,326/mo fails the 200% FPL limit and SNAP is $0." +us,scenario_121,snap,gemini-3.6-flash,llm_error,age_disability,False,"Exempted the head from the gross income test and took a $2,865/mo excess medical deduction. Neither applies because the head is not a SNAP elderly/disabled member, and gross income of $3,326/mo exceeds the 200% FPL BBCE limit, giving $0." +us,scenario_121,snap,gemini-3.7-flash,llm_error,age_disability,False,"Treated the head as SNAP-disabled, exempt from the gross test and entitled to the excess medical deduction. The head lacks SSI or SSDI receipt and does not qualify, so $39,909 of gross income fails the 200% FPL limit and SNAP is $0." +us,scenario_121,snap,glm-5.3,llm_error,age_disability,False,"Exempted the household from the gross income test as having a disabled member. The head is not SNAP-disabled under the benefit-receipt definition, so countable gross income (actually $39,909 including child support, not $38,709) exceeds the 200% FPL limit and SNAP is $0." +us,scenario_121,snap,gpt-5.5,llm_error,age_disability,False,"Applied the disabled-household rules, including the medical deduction and net-income-only eligibility. The head is not a SNAP disabled member, so the gross income test applies, and $3,326/mo exceeds the 200% FPL limit, giving $0 rather than the $298/mo maximum." +us,scenario_121,snap,gpt-5.6-sol,llm_error,age_disability,False,"Let the disabled head deduct medical costs and bypass the gross income test. SNAP's disabled-member definition requires receipt of SSI, SSDI, or similar benefits, which the head lacks, so gross income of $39,909 fails the 200% FPL limit and SNAP is $0." +us,scenario_121,snap,gpt-5.6-terra,llm_error,age_disability,False,"Treated the household as a SNAP disabled household and zeroed out net income with medical expenses. Without qualifying disability-benefit receipt the head is not SNAP-disabled, so the gross income test applies and $3,326/mo exceeds the 200% FPL limit, giving $0." +us,scenario_121,snap,gpt-6-astra,llm_error,age_disability,False,"Took a $32,180 disability-related medical deduction and skipped the gross income test. The head does not qualify as a SNAP elderly/disabled member, so the medical deduction is unavailable, and gross income of $39,909 exceeds the 200% FPL limit, giving $0." +us,scenario_121,snap,gpt-6-luna,llm_error,age_disability,False,"Applied allowable medical deductions for a disabled member and awarded the maximum allotment. The head is not SNAP-disabled, so the gross income test applies, and $3,326/mo exceeds SC's 200% FPL BBCE limit, giving $0." +us,scenario_121,snap,gpt-6-sol,llm_error,age_disability,False,"Relied on the disabled-household medical deduction to zero out net income. The head lacks SSI or SSDI receipt and is not a SNAP disabled member, so gross income of $39,909 fails the 200% FPL gross limit and SNAP is $0." +us,scenario_121,snap,gpt-6.1-sol,llm_error,age_disability,False,"Stated that disability permits the net-income-only test. The head's generic disability benefits and above-SGA wages do not make them SNAP-disabled, so the gross income test applies, and $39,909 exceeds the 200% FPL limit, giving $0." +us,scenario_121,snap,grok-4.6,llm_error,age_disability,False,"Had the disabled one-person unit skip the gross income test and take the excess medical deduction. The head is not a SNAP disabled member, so the gross income test applies, and $3,326/mo exceeds the 200% FPL BBCE limit, giving $0." +us,scenario_121,snap,grok-4.7,llm_error,age_disability,False,"Waived the gross income test as a disabled unit and took a $31,780 medical deduction. The head is not SNAP-disabled under the benefit-receipt definition, so neither applies, and gross income of $39,909 exceeds the 200% FPL limit, giving $0." +us,scenario_121,snap,grok-build-0.1,llm_error,age_disability,False,"Applied only the net income test because of the disabled head. The head does not meet SNAP's disabled-member definition, so the gross income test applies, and $3,326/mo exceeds the 200% FPL limit, giving $0 rather than a $3,600 maximum allotment." +us,scenario_121,snap,inkling,llm_error,age_disability,False,"Treated the household as SNAP-disabled and used the out-of-pocket medical deduction to reach $0 net income. The head lacks qualifying SSI or SSDI receipt, so the gross income test applies, and $39,909 exceeds the 200% FPL limit, giving $0." +us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there is no answer to compare with the $0 result from the failed gross income test." +us,scenario_121,snap,kimi-k3,llm_error,age_disability,False,"Treated the head as disabled for SNAP, applying the uncapped medical deduction and skipping the gross test. The head is not a SNAP disabled member, so gross income of $3,325.75/mo fails the 200% FPL BBCE limit and SNAP is $0. It also cited an outdated $3,000 resource limit." +us,scenario_121,snap,muse-wonder,llm_error,age_disability,False,"Took a $2,682/mo medical deduction and never tested $3,326/mo of gross income against the gross limit. The head is not SNAP-disabled, so the gross income test applies and the 200% FPL limit (about $2,608/mo) is exceeded, giving $0." us,scenario_121,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model raised the itemized-deduction question and then dismissed it — ""the standard deduction likely exceeds itemized here"" — when the $32,600 of medical outlays less the 7.5%-of-AGI floor leaves about $30,000 of deduction against a $16,100 standard deduction, zeroing federal taxable income and therefore SC's starting point. It then built its $400 on a $14,600 standard deduction (the 2024 federal figure, not 2026's $16,100), a fabricated $4,610 SC personal exemption (SC grants no exemption for the filer, only for dependents), and a flat 6.2% rate in place of SC's 0%/3%/top-rate schedule." us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model gave no derivation, but $686 is exactly 6.2% × $11,065, i.e. wages of $25,665 reduced by a $14,600 standard deduction and taxed at a single flat rate. That path never claims the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor exceeds the standard deduction by roughly $14,000 and drives SC taxable income to zero, and it also ignores SC's zero-rate bracket on the first ~$3,700 and the 3% bracket above it." @@ -9950,99 +10079,101 @@ us,scenario_121,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambigu us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"The model taxed $38,709 of gross income less a $15,100 standard deduction, never applying the $32,600 medical expense itemized deduction that after the 7.5%-of-AGI floor exceeds that standard deduction by roughly $14,000 and zeroes the SC base. It compounded this with a fabricated bracket schedule containing 6.4% and 6.5% tiers (SC has only a zero-rate bracket, a 3% bracket, and one top rate) and then cut its own $1,395 result to $574 with an unquantified ""exemptions/credits"" adjustment it never identified." us,scenario_121,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model used the correct 2026 federal standard deduction of $16,100 and correctly kept disability benefits out of the base, but stated that ""no SC subtractions"" apply and never itemized: the $32,600 of medical expenses less the 7.5% floor exceeds its own $25,665 AGI, so federal taxable income is $0, not $9,565. With a zero starting point, SC's 0%/3% schedule produces $0 rather than $180." us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model reduced wages by a $14,600 standard deduction while ignoring the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor eliminates taxable income entirely. It then applied an invented four-tier SC schedule (0%/3%/4%/5.5%) that does not exist — SC has a zero-rate bracket, a 3% bracket, and a single top rate — and its $94.26 does not even follow from that schedule, which would yield about $314 on its own $11,065 base." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its deductions stopped at $26,200 (standard $20,200 plus the $6,000 senior deduction). It left out the 2026 $1,000 charitable deduction for non-itemizers, so it worked out $4,867 instead of $4,746.66. It then submitted $6,552, which contradicts its own computation." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted only 50% of Social Security ($16,000) as taxable. Provisional income of $61,459 makes the 85% cap of $27,200 taxable. It also made up a 'disabled' additional standard deduction, then cut its own $3,668 tentative tax to $1,065 by citing credits that don't exist." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the senior, blind and disability deductions wipe out all taxable income. In fact $70,646 of AGI less $27,200 of deductions leaves $43,446.18 taxable, which carries $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted only the $20,200 standard deduction. It left out the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, which put taxable income at $50,446 instead of $43,446.18 and pushed part of the qualified dividends into the 15% band." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used outdated $15,000/$2,000/$2,000 standard deduction amounts and missed the $1,000 non-itemizer charitable deduction, reaching about $5,011. It then cut that to $1,971 by counting the age/blind additional amounts a second time." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income of about $43,546 was near the correct figure, but its bracket arithmetic failed. Tax on $41,722 of ordinary income is $1,240 + 12% × $29,322 ≈ $4,759, not $2,336, and it charged 15% on qualified dividends that fall entirely in the 0% band. It also used an inflated $17,000 base standard deduction and left out the $1,000 non-itemizer charitable deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and applied the old 10%/15%/25% brackets, a personal exemption, miscellaneous itemized deductions and an $11,400 standard deduction. OBBBA made the TCJA structure permanent, so 2026 uses the 10%/12% brackets, the $20,200 aged/blind standard deduction, the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $1,824 of qualified dividends as part of the $3,728 of non-qualified dividends, which understated AGI at $68,822. It also left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income came out at $48,622 instead of $43,446.18." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly applied the $20,200 standard deduction plus the $6,000 senior deduction. It left out the $1,000 charitable deduction that single non-itemizers can claim starting in 2026, so its taxable income was $1,000 too high ($44,446) and its tax $120 too high." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used an outdated $19,200 standard deduction and left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That gave taxable income of $51,446 instead of $43,446.18 and taxed part of the qualified dividends at 15%." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a personal exemption, deductible miscellaneous employee expenses, an itemized state income tax deduction and a 15% second bracket. OBBBA made the TCJA brackets permanent and kept those items repealed, so the correct path is the $27,200 deduction stack and the 10%/12% brackets." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It treated 2026 as a TCJA-sunset year, with a $5,300 personal exemption, 2%-floor miscellaneous deductions and 10%/15% brackets. Under OBBBA the correct path is the standard deduction plus the $6,000 senior deduction and $1,000 non-itemizer charitable deduction, taxed at 10%/12%." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It deducted only the $20,200 standard deduction and left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That gave taxable income of $50,446 and pushed $996 of qualified dividends into the 15% band." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired, claiming $18,126 of itemized deductions plus a $5,200 personal exemption and taxing income in a 25% bracket. OBBBA kept the TCJA brackets and eliminated personal exemptions, so the correct path is the $27,200 deduction stack and 10%/12% rates." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its derivation doesn't hold together. It overstated AGI at $71,468, subtracted a made-up $29,600 standard deduction but still reported $69,455 of taxable income, and then also subtracted itemized deductions, which is not allowed alongside the standard deduction. The correct taxable income is $43,446.18 from the $27,200 stack of standard, senior and non-itemizer charitable deductions." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $6,300 implies roughly $50,000+ of taxable income, which matches leaving out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, or applying TCJA-sunset rules, instead of taxing $43,446.18 at 10%/12%." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules, with $2,097 of 2%-floor employee expenses, a $5,150 personal exemption and post-sunset brackets, and taxed qualified dividends at 15%. Under OBBBA, taxable income is $43,446.18 after the standard, senior and non-itemizer charitable deductions, and the qualified dividends are taxed at 0%." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It computed about $4,917 of tax and then reduced it to $0 by pointing to nonrefundable credits and deduction rules that don't apply. This filer has no available nonrefundable credit (the elderly/disabled credit is fully phased out at this AGI)." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It applied only the aged/blind standard deduction. It left out the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, and its $5,994 matches taxable income of about $51,600 instead of $43,446.18." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no numbers, and its submitted $6,712 contradicts the 'value = 6979' in its own explanation. Both figures come from itemizing instead of taking the $20,200 standard deduction plus the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used pre-OBBBA 2025 amounts ($15,000 + $2,000 + $2,000 = $19,000) and left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That gave $51,646 of taxable income, and it then taxed the qualified dividends at 15% when they belong in the 0% band." -us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It dropped the $1,824 of qualified dividends from provisional income and AGI, which understated taxable Social Security at $26,289.75 instead of the $27,200 cap. It also left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, and it used a $19,570 standard deduction instead of $20,200." -us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as 0.85 × (provisional income − $34,000) = $23,340, skipping the $4,500 add-on. The formula actually gives $27,840, which is capped at $27,200. It also used $3,300 for the age/blind amounts instead of 2 × $2,050 and a $12,200 10% bracket instead of $12,400, which understated tax at $4,383." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age/blind deductions fully offset income. AGI of $70,646.18 (including $27,200 of taxable Social Security) less $27,200 of deductions leaves $43,446.18 taxable and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It wrongly treated tax-exempt income and implied itemized deductions as wiping out the liability. Tax-exempt interest only feeds the Social Security provisional-income test, itemized deductions ($14,573) are smaller than the standard deduction, and taxable income is $43,446.18." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA-sunset' rules: $17,023 of itemized deductions including 2%-floor employee expenses, a $5,300 personal exemption and a 15% bracket. OBBBA made TCJA permanent, so the correct calculation uses the $27,200 standard/senior/charitable deduction stack and 10%/12% brackets." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly included the senior deduction and the $1,000 non-itemizer charitable deduction, but its total deductions were $27,100 instead of $27,200. It used $2,000 instead of $2,050 for each aged/blind additional amount, so taxable income came out at $43,546 and tax about $12 too high." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $4,957 implies about $1,750 less in deductions than the correct $27,200. It never mentioned the $1,000 non-itemizer charitable deduction ($120 of the gap) and understated the 2026 standard/aged/blind amounts for the rest." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard, age/blind and $6,000 senior deductions but left out the $1,000 charitable deduction for non-itemizers that starts in 2026. That gave taxable income of about $44,465 instead of $43,446.18, roughly $120 too much tax." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It wrongly said income falls below filing thresholds because there is no earned income. Pension, IRA, dividend and 85% Social Security income produce $70,646 of AGI and $43,446.18 of taxable income." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used TCJA-sunset projections: a personal exemption, 2%-floor miscellaneous deductions, a state income tax itemized deduction and a 15% bracket. Under OBBBA, the standard deduction plus the $6,000 senior deduction and $1,000 non-itemizer charitable deduction apply, and tax is computed at 10%/12%." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a personal exemption, 2%-floor miscellaneous employee expenses and a 15% ordinary bracket, which are TCJA-sunset rules. OBBBA made those changes permanent, so the $27,200 OBBBA deduction stack and the 10%/12% rates apply." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied the standard, aged/blind and $6,000 senior deductions but left out the $1,000 charitable deduction for single non-itemizers that starts in 2026. That left taxable income at $44,396 instead of $43,446.18, and it also used slightly off $16,150 base and $12,250 bracket figures." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA reverted in 2026, itemizing state income tax and 2%-floor employee expenses and applying 10%/15% brackets inflated from 2017. OBBBA kept the TCJA structure, so taxable income is $43,446.18 after the standard, senior and non-itemizer charitable deductions, taxed at 10%/12%." -us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a personal exemption, deductible miscellaneous expenses, state tax in SALT and a 15% bracket. Under OBBBA, the correct path is the $20,200 standard deduction plus the $6,000 senior deduction and $1,000 non-itemizer charitable deduction, taxed at 10%/12%." -us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It admitted taxable income was positive but claimed personal exemptions and credits reduce the tax to zero. Personal exemptions are $0 under OBBBA and no nonrefundable credit applies, so $43,446.18 of taxable income gives $4,746.66." -us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its deductions stopped at $26,200 (standard plus senior) and left out the $1,000 charitable deduction for non-itemizers that starts in 2026, so its reasoning reached $4,866.64. It then submitted $3,491.46, which matches neither its own work nor the correct $4,746.66." -us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. It also wrongly counted the $1,447 long-term capital loss as preferentially taxed income, and its submitted $9,505 contradicts its own computed $5,544." -us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left Social Security out of AGI even though provisional income of $61,459 makes $27,200 taxable. It also applied an elderly/disabled credit that phases out completely above $17,500 of AGI, and its final $5,065 is larger than its own $3,933.60 tentative tax." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its worked computation left out OBBBA's $1,000 non-itemizer charitable deduction, so it reached $44,446 of taxable income and $4,867 of tax instead of $43,446.18 and $4,746.66. It then submitted $6,552, a figure that contradicts its own computation." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It taxed only 50% of Social Security ($16,000), even though provisional income of $61,459 makes the 85% cap ($27,200) apply. It invented a $21,700 base and a 'disabled' add-on to the standard deduction and omitted the $6,000 senior deduction. It then cut its $3,668 tentative tax to $1,065 using nonexistent credits." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed the deductions wipe out all tax. In fact, $70,646 of AGI minus $27,200 of deductions (standard, aged/blind, senior and charitable) leaves $43,446 of taxable income, and the ordinary portion is taxed at 10%/12% for $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It used the correct $20,200 standard deduction but left out the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. That raised taxable income to $50,446 and pushed part of the qualified dividends into the 15% rate." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used a roughly $25,000 deduction (outdated $15,000 base, no $1,000 charitable deduction) to get $45,646 of taxable income and about $5,011 of tax. It then arbitrarily cut taxable income to about $40,000 and reported an unsupported $1,971." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $17,000 base standard deduction instead of $16,100 and left out the $1,000 non-itemizer charitable deduction. It also miscomputed the bracket tax: it got $2,336 on $41,722 of ordinary income, when 10%/12% yields about $4,759, and it put tax on qualified dividends that fall in the 0% band." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026. On that basis it applied a personal exemption, pre-TCJA 10%/15%/25% brackets, and itemized deductions that included miscellaneous employee expenses above a 2% floor. Under OBBBA, 2026 has the $20,200 standard deduction (with aged/blind add-ons), the $6,000 senior deduction and the $1,000 charitable deduction, with 10%/12% brackets." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of AGI (counting only the $3,728 of non-qualified dividends). It also omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, which left $48,622 of taxable income instead of $43,446.18." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly applied the $20,200 standard deduction and the $6,000 senior deduction but left out the $1,000 non-itemizer charitable deduction available in 2026. That gave $44,446 of taxable income and $4,867 of tax instead of $43,446.18 and $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used an outdated $15,300 base and a $19,200 standard deduction, and omitted both the $6,000 senior deduction and the $1,000 charitable deduction. That produced $51,446 of taxable income and pushed the qualified dividends into the 15% rate." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: itemized deductions that included miscellaneous expenses and state income tax, a personal exemption, and the 15% bracket. OBBBA keeps the TCJA standard deduction with aged/blind add-ons, adds the $6,000 senior and $1,000 charitable deductions, and keeps the 10%/12% brackets for 2026." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It treated 2026 as a TCJA-sunset year, with a $5,300 personal exemption, miscellaneous itemized deductions above 2% of AGI, an uncapped state income tax deduction and the 15% bracket. The correct approach uses OBBBA's $27,200 of standard-type deductions and the 10%/12% brackets." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the $20,200 standard deduction but left out the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Taxable income came out at $50,446 instead of $43,446.18, and part of the qualified dividends was taxed at 15%." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expiration, combining $18,126 of itemized deductions with a $5,200 personal exemption and 15%/25% brackets. Under OBBBA, 2026 uses the $20,200 standard deduction plus the $6,000 senior and $1,000 charitable deductions, leaving $43,446.18 taxed at 10%/12%." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its arithmetic is incoherent. It overstated AGI at $71,468, applied an inflated $29,600 standard deduction yet reported $69,455 of taxable income, and then also subtracted $10,808 of itemized deductions on top of the standard deduction. The correct path is $70,646 AGI minus $27,200 of deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $6,300 is about $1,550 too high, which matches treating 2026 as a TCJA-sunset year (personal exemption, itemizing, 15% bracket) or leaving out the $6,000 senior and $1,000 charitable deductions. The correct deductions bring taxable income to $43,446.18." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: itemized deductions that included $2,097 of miscellaneous employee expenses, a $5,150 personal exemption, and a 15% rate on qualified dividends. Under OBBBA, 2026 has no personal exemption or miscellaneous deductions; the $27,200 of standard-type deductions applies and the qualified dividends fall in the 0% band." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It computed about $4,917 of tax and then reduced it to zero using nonexistent nonrefundable credits. No nonrefundable credit applies, because the elderly/disabled credit is fully phased out at this AGI." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took only the standard deduction with aged/blind add-ons and omitted OBBBA's $6,000 senior deduction and $1,000 non-itemizer charitable deduction, so taxable income was about $7,000 too high." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It itemized under post-sunset rules instead of taking OBBBA's $20,200 standard deduction plus the $6,000 senior and $1,000 charitable deductions. Its reasoning ends with value = 6979, which contradicts the $6,712 it submitted." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used 2025 pre-OBBBA amounts ($15,000 base plus $2,000 x 2 = $19,000) and omitted the $6,000 senior and $1,000 charitable deductions. That gave $51,646 of taxable income and put the qualified dividends at 15% instead of 0%." +us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of both provisional income and AGI, which cut taxable Social Security below the $27,200 cap. It also omitted the $6,000 senior deduction and the $1,000 charitable deduction and used a $15,450 base deduction." +us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as 0.85 x (provisional income - $34,000) = $23,340, leaving out the $4,500 base amount, so it missed that the 85% cap of $27,200 applies. It also used a $3,300 aged/blind add-on instead of $4,100 and a $12,200 bracket edge instead of $12,400." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and aged/blind deductions fully offset income. In fact, after $27,200 of total deductions, $43,446.18 of taxable income remains, and the $41,622 ordinary portion is taxed for $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated tax-exempt income and itemized deductions as eliminating the liability. In fact, $70,646 of AGI (including $27,200 of taxable Social Security) minus $27,200 of deductions leaves $43,446.18 of taxable income and $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: about $17,023 of itemized deductions that included miscellaneous expenses, a $5,300 personal exemption and pre-TCJA brackets. Under OBBBA, 2026 has the $20,200 standard deduction plus the $6,000 senior and $1,000 charitable deductions, and 10%/12% brackets." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It identified the correct deduction components but totaled them $100 short ($27,100 instead of $16,100 + $4,100 + $6,000 + $1,000 = $27,200). That gave $43,546 of taxable income and $4,759 of tax instead of $43,446.18 and $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $4,957 is $210 too high, which means taxable ordinary income was about $1,750 too high. That matches leaving out the $1,000 non-itemizer charitable deduction and understating the $20,200 standard deduction with aged/blind add-ons." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard, aged/blind and $6,000 senior deductions but left out the $1,000 non-itemizer charitable deduction, so it got about $44,465 of taxable income and $4,869 of tax instead of $43,446.18 and $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed taxable income falls below filing thresholds. In fact, $70,646 of AGI minus $27,200 of deductions leaves $43,446.18 of taxable income, taxed at 10%/12% for $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It assumed TCJA sunset, using about $12,400 as the standard deduction, itemizing with miscellaneous expenses and Minnesota income tax, a $5,300 personal exemption and a 15% bracket. Under OBBBA, 2026 has a $20,200 standard deduction plus the $6,000 senior and $1,000 charitable deductions, taxed at 10%/12%." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized with miscellaneous employee expenses above a 2% floor and added a personal exemption under a 15% bracket. All of these assume TCJA expired, whereas OBBBA keeps the TCJA standard deduction structure and adds the $6,000 senior and $1,000 charitable deductions for 2026." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $16,150 base instead of $16,100 and left out the $1,000 non-itemizer charitable deduction, so taxable income came out at $44,396. It also ended the 10% bracket at $12,250 instead of $12,400." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied a TCJA reversion: itemizing with miscellaneous expenses and state income tax, and using 10%/15% brackets inflated from 2017. OBBBA keeps the $20,200 aged/blind standard deduction, adds the $6,000 senior and $1,000 charitable deductions, and keeps the 10%/12% brackets for 2026." +us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used post-sunset rules: about $20,100 of itemized deductions that included Minnesota tax and miscellaneous expenses, a $5,050 personal exemption and the 15% bracket. OBBBA eliminates the personal exemption and miscellaneous deductions and provides $27,200 of standard-type deductions in 2026." +us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It produced no answer and no explanation for federal_income_tax_before_refundable_credits. +us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed personal exemptions and credits fully offset tax. In 2026 there is no personal exemption and no nonrefundable credit applies, so $43,446.18 of taxable income produces $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It reached the correct $43,446 of taxable income but reported a rounded $4,750 instead of computing $1,240 + 12% x $29,222.18 on $41,622.18 of ordinary income, which is $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its reasoning left out the $1,000 non-itemizer charitable deduction and reached $44,446 of taxable income and $4,866.64 of tax. It then submitted $3,491.46, which contradicts its own computation." +us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $20,050 standard deduction and omitted the $6,000 senior and $1,000 charitable deductions. It also treated the negative $1,447 long-term capital gain as preferentially taxed income, reaching $5,544. It then submitted an unrelated $9,505." +us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It excluded taxable Social Security from AGI, although 85% ($27,200) is includable, and omitted the $6,000 senior deduction. It also claimed an elderly/disabled credit that is fully phased out at this AGI, and its final $5,065 exceeds its own $3,933.60 tentative tax." us,scenario_122,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_122,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 73, blind, and disabled as automatic categorical eligibility and conflated Medicare Savings Programs such as QMB and SLMB with full Medicaid eligibility. It also dismissed resource limits despite the disclosed $44,000 in bank assets and $177,000 in stock assets; the head qualified through none of Minnesota’s Medicaid pathways." us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly found income far above the 138% FPL MAGI limit, then reversed that result solely because the head was elderly, blind, and disabled. Those characteristics do not automatically confer Medicaid eligibility, and the head qualified through no alternative category." us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wage earnings were provided and that payroll tax therefore equals zero, then contradicted that reasoning by imposing an unsupported $1,270.64 tax on unspecified ""taxable income."" Retirement and investment income is not an employee Social Security or Medicare tax base." us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable payroll_tax output or explanation, violating the required structured-output contract." us,scenario_122,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It said the Minnesota Social Security subtraction was phased out at this income. In fact AGI of $70,646 is below the single-filer phase-out threshold, so the full $27,200 is subtracted. Without that subtraction it got about $52,000 of taxable income in the 6.8% bracket instead of $19,169 taxed at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The method was right (AGI minus the $27,200 Social Security subtraction, standard deduction and charitable subtraction, taxed at 5.35%), but two inputs were too high. It used a $19,500 standard deduction instead of $19,300, and a charitable subtraction of $5,154 (50% × ($10,808 − $500)) instead of the engine's $4,977.18. That gave $18,792 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $27,200 of taxable Social Security out of the income base and then declared the tax zero without doing the math. Even its own figures ($43,446 of income minus a $13,175 deduction) leave about $30,000 taxable. The correct path leaves $19,169 taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed a pension subtraction that does not apply to this filer and assumed the deductions wipe out all tax. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 remains taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It got close to the right path (about $19,442 taxable and about $1,040 of tax). It then wrongly applied Minnesota's elderly/disabled subtraction, which is fully phased out for this filer because AGI and nontaxable Social Security exceed the limits. It also applied a long-term care credit even though there are no long-term care premiums. Together these cut the tax to $0 instead of $1,025.54." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It asserted that the Social Security subtraction plus deductions push taxable income to near zero, and it made up nonrefundable credits for age and blindness. Subtracting $27,200 of Social Security, the $19,300 standard deduction and the $4,977.18 charitable subtraction from $70,646 still leaves $19,169 taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security subtraction as only partial, when the full $27,200 applies below the phase-out threshold. It also understated the age/blind add-ons (about $3,300 instead of the amount that brings the deduction to $19,300) and left out the non-itemizer charitable subtraction. Those errors produced about $44,000 of taxable income spanning two brackets." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The Social Security subtraction ($27,200) and the standard deduction ($19,300) were both correct. The only error was the charitable subtraction: it used 50% × ($10,808 − $500) = $5,154 instead of the engine's $4,977.18. That understated taxable income by $176.82 ($18,992 vs. $19,169)." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income, even though Minnesota has started from federal AGI since 2019. It made up a $15,000 Minnesota pension/IRA subtraction that does not exist for this filer. It also itemized with unreimbursed employee expenses instead of taking the $19,300 age/blind standard deduction plus the charitable subtraction. The result was $10,010 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left taxable Social Security out of federal AGI ($43,446) and then still talked about subtracting Social Security again. It also made up an elderly/disability credit to reach zero. In fact, $19,169 of taxable income remains after the $27,200 subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the full $27,200 of Social Security but left out the $4,977.18 non-itemizer charitable subtraction. It also used a $19,400 standard deduction instead of $19,300, which gave $24,046 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It skipped the full $27,200 Social Security subtraction. It also subtracted a $5,300 personal exemption, which Minnesota no longer allows for the filer (only dependents get exemptions), and used a $15,200 standard deduction with no age/blind add-ons. The resulting $50,146 of taxable income fell partly in the 6.8% bracket." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted a personal exemption that Minnesota does not allow for the filer and $17,023 of itemized deductions. It never took the $27,200 Social Security subtraction and never compared against the $19,300 age/blind standard deduction plus the charitable subtraction. That gave $48,529 of taxable income in two brackets instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It correctly subtracted $27,200 of Social Security but understated the age/blind standard deduction ($18,500 vs. $19,300). It also left out the $4,977.18 non-itemizer charitable subtraction, leaving $24,946 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It made up a $5,000 retirement subtraction, used an $18,500 standard deduction instead of $19,300, and left out the $4,977.18 charitable subtraction. The errors partly cancel, landing at $19,946 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It never subtracted the $27,200 of taxable Social Security, which Minnesota excludes in full at this AGI, and it left out the charitable subtraction. It also taxed the excess at 7.05%, but Minnesota's second bracket rate is 6.8%. The result was $51,946 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"A $2,420 tax implies about $42,500 of Minnesota taxable income, spanning the 5.35% and 6.8% brackets. That is consistent with not taking the full $27,200 Social Security subtraction. The correct derivation leaves $19,169 taxed entirely at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied only a partial Social Security subtraction. In fact AGI of $70,646 is below the single-filer phase-out threshold, so the full $27,200 is subtracted. The leftover income pushed taxable income into the 6.8% bracket instead of leaving $19,169 at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied only a partial Social Security subtraction even though the full $27,200 is subtracted at this AGI, and it left out the charitable subtraction. That gave $51,665 of taxable income in two brackets instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"A $1,111 tax implies about $20,766 of taxable income at 5.35%, roughly $1,600 above the correct $19,169. That is consistent with taking the Social Security subtraction but understating the combined $19,300 age/blind standard deduction and $4,977.18 charitable subtraction." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It correctly subtracted $27,200 of Social Security but used an $18,270 standard deduction instead of $19,300. It also left out the $4,977.18 non-itemizer charitable subtraction, leaving $25,176 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It submitted $2,389, which contradicts its own explanation's figure of $1,308. Neither number reflects the correct path: $70,646 AGI minus the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction. The $2,389 figure implies taxable income in the 6.8% bracket, consistent with an incomplete Social Security subtraction." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the full $27,200 of Social Security but used a $19,000 standard deduction instead of $19,300. It also left out the $4,977.18 non-itemizer charitable subtraction, leaving $24,446 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used made-up rates of 2.85% and 4.80%; Minnesota's lowest rate is 5.35%. It also chose $14,926 of itemized deductions over a $14,600 standard deduction that left out the age and blindness add-ons. The correct choice is the $19,300 standard deduction plus the $4,977.18 charitable subtraction, giving $19,169 taxable." -us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It reached the correct $43,446 Minnesota base after the Social Security subtraction. However, it used an $18,560 standard deduction instead of $19,300 and left out the $4,977.18 non-itemizer charitable subtraction, leaving $24,886 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed senior/disabled subtractions wipe out taxable income. In fact the elderly/disabled subtraction is fully phased out at this income. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 remains taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave a zero tax with no computation. The correct path leaves $19,169 of Minnesota taxable income after the Social Security subtraction, the age/blind standard deduction and the charitable subtraction, producing $1,025.54 at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It added back the $2,013 of tax-exempt interest as a Minnesota addition. The facts do not identify it as non-Minnesota bond interest, and the engine applies no such addition. Adding it raised taxable income to about $20,450 instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"About $49,900 of taxable income means it did not apply the full $27,200 Social Security subtraction or the $4,977.18 charitable subtraction. Both apply in full at this AGI, and the correct base is $19,169 taxed entirely at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It added the $2,013 of tax-exempt interest back as a Minnesota addition, which the engine does not do for these facts. That raised taxable income to $21,055. Without the addition, its subtractions land near the correct $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"A $2,640 tax implies about $45,800 of taxable income reaching the 6.8% bracket, consistent with not taking the full $27,200 Social Security subtraction. It also left out the charitable subtraction. The correct base is $19,169 taxed entirely at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It implicitly used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of the engine's $4,977.18. That understated taxable income at $18,892 instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took the Social Security subtraction and the age/blind standard deduction but left out the $4,977.18 non-itemizer charitable subtraction. That left $23,265 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $18,342 of taxable income implies about $25,100 of combined standard deduction and charitable subtraction. The correct total is $24,277.18 ($19,300 standard deduction + $4,977.18 charitable subtraction), so it overstated the deductions and understated taxable income relative to $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It correctly subtracted $27,200 of Social Security but left out the $4,977.18 non-itemizer charitable subtraction. It also used a $19,400 standard deduction instead of $19,300, leaving $24,046 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed age-based exemptions bring taxable income to zero. Minnesota has no such exemption at this income level. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 is taxed at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $19,600 standard deduction instead of $19,300 and left out the $4,977.18 non-itemizer charitable subtraction. That gave about $23,844 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"After the Social Security subtraction and the standard deduction, it did not apply the $4,977.18 non-itemizer charitable subtraction. It ended up with about $22,000 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It added back the $2,013 of tax-exempt interest, which the engine does not treat as a Minnesota addition for these facts. It also used a $19,400 standard deduction and a $5,154 charitable subtraction instead of $19,300 and $4,977.18, giving $20,905 of taxable income instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It never took the $27,200 Social Security subtraction. It itemized $17,023 instead of taking the larger $19,300 age/blind standard deduction plus the $4,977.18 charitable subtraction. That produced $53,623 of taxable income in the 6.8% bracket." -us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It kept taxable income near $50,000 by not taking the full $27,200 Social Security subtraction, which applies in full below the phase-out threshold. It also left out the charitable subtraction, putting income in the 6.8% bracket instead of leaving $19,169 at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there was no answer to score against the $1,025.54 result." -us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $5,330 personal exemption, but Minnesota allows exemptions only for dependents, not the filer. It also left out the $4,977.18 non-itemizer charitable subtraction. Those errors largely cancel, giving $18,690 instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It relied on a senior age-65 subtraction that is fully phased out at this income and concluded taxable income is zero. In fact $19,169 remains after the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction." -us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It itemized $17,367 against a $15,300 standard deduction that left out the age and blindness add-ons. The correct $19,300 standard deduction plus the $4,977.18 non-itemizer charitable subtraction totals $24,277 and wins. That left $26,079 taxable instead of $19,169." -us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It took no Social Security subtraction and added back tax-exempt interest and pension income. It used a $14,600 standard deduction with no age/blind add-ons. It then cut the liability with the property tax refund, which is a separate refund and not a nonrefundable income tax credit." -us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used the wrong federal AGI ($52,092 instead of $70,646) and an invented $30,100 standard deduction, and it never took the Social Security or charitable subtractions. It also made an arithmetic error: $21,992 at 5.35% is $1,177, not $1,450." +us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It claimed the Minnesota Social Security subtraction was phased out at this income, but AGI of $70,646 is below the single threshold, so the full $27,200 is subtracted. It also itemized instead of taking the $19,300 aged/blind standard deduction plus the $4,977 charitable subtraction. Its ~$52,000 of taxable income spilled into the 6.8% bracket instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The structure was right: full Social Security subtraction, aged/blind standard deduction, and the non-itemizer charitable subtraction. But it used a $19,500 standard deduction instead of $19,300. It also computed the charitable subtraction as a plain 50% of contributions over $500 ($5,154), while the engine's figure is $4,977.18. Together these understated taxable income by $377 ($18,792 vs $19,169)." +us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $27,200 of taxable Social Security out of the income base and used a $13,175 standard deduction, which still leaves about $30,000 of positive taxable income. It then declared $0 tax without applying Minnesota's 5.35% rate. The correct path yields $19,169 of taxable income and $1,025.54 of tax." +us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction, the Social Security subtraction and a pension exemption wipe out Minnesota taxable income. Minnesota has no general private-pension subtraction, and after the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 remains taxable at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It reached a taxable income of ~$19,442 and tax of ~$1,040, then zeroed it out with a 'senior/disabled subtraction' and long-term-care credit. Minnesota's elderly/disabled subtraction requires single AGI below roughly $33,700, so it is unavailable at $70,646 AGI, and no long-term-care expense was given." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It never computed taxable income and asserted that the Social Security subtraction plus deductions drive it to about zero. With $43,446 of income after the Social Security subtraction, minus the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 remains taxable at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the Minnesota Social Security subtraction as only partial, but the full $27,200 is subtracted at this AGI. It also omitted the non-itemizer charitable subtraction, inflating taxable income to ~$44,000 and reaching the 6.8% bracket instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It correctly applied the full $27,200 Social Security subtraction and the $19,300 standard deduction, but set the charitable subtraction at $5,154, a plain 50% of contributions over $500. The engine's subtraction is $4,977.18, so taxable income came out $177 too low ($18,992 vs $19,169)." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income rather than federal AGI, and invented a $15,000 Minnesota pension/IRA subtraction for seniors. It also itemized with unreimbursed employee expenses instead of taking the $19,300 aged/blind standard deduction plus the $4,977 charitable subtraction. This produced $10,010 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It claimed that deductions, subtractions and an 'elderly/disabled credit' push taxable income to about zero. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 remains, taxed at 5.35% for $1,025.54. The elderly/disabled subtraction is barred by its AGI limit." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It applied the full Social Security subtraction but omitted Minnesota's non-itemizer charitable subtraction ($4,977.18) and used a $19,400 standard deduction instead of $19,300. That left $24,046 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It skipped the full $27,200 Social Security subtraction and subtracted a nonexistent personal exemption; Minnesota allows exemptions only for dependents. It also omitted the aged/blind standard-deduction additions and the charitable subtraction, taxing ~$50,146 at 5.35%/6.8% instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It never subtracted the $27,200 of taxable Social Security and claimed a personal exemption Minnesota does not allow for the filer. It also itemized ($17,023) instead of taking the $19,300 aged/blind standard deduction plus the $4,977 charitable subtraction, taxing $48,529 instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied the full Social Security subtraction but used an $18,500 standard deduction instead of $19,300. It also omitted the non-itemizer charitable subtraction of $4,977.18, leaving $24,946 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It invented a $5,000 Minnesota retirement subtraction that does not apply here and used an $18,500 standard deduction instead of $19,300. It also omitted the $4,977.18 charitable subtraction. Its $19,946 taxable income is near the correct figure only because these errors partly offset." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It omitted the full $27,200 Social Security subtraction and the non-itemizer charitable subtraction, and taxed $51,946 using a 7.05% second-bracket rate, while Minnesota's second rate is 6.8%. The correct taxable income is $19,169, taxed entirely at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $2,420 implies about $45,000 of taxable income, which reflects no Minnesota subtraction for the $27,200 of federally taxable Social Security and no charitable subtraction. With both applied plus the $19,300 standard deduction, taxable income is $19,169 and tax is $1,025.54." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied only a 'partial' Social Security subtraction, but AGI is below the single phaseout threshold, so the full $27,200 is subtracted. It also omitted the charitable subtraction, pushing income into the 6.8% bracket instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It treated the Social Security subtraction as partial and taxed $51,665, but the full $27,200 is subtracted at this AGI. With the $19,300 standard deduction and the $4,977 charitable subtraction, taxable income is $19,169, all at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $1,111 implies about $20,770 of taxable income at 5.35%, roughly $1,600 above the correct $19,169. That is consistent with understating the standard deduction or the non-itemizer charitable subtraction ($4,977.18)." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It applied the full Social Security subtraction but used an $18,270 standard deduction instead of $19,300. It also omitted the $4,977.18 non-itemizer charitable subtraction, leaving $25,176 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It submitted $2,389, which contradicts its own explanation ending in $1,308. Neither figure applies the $4,977.18 non-itemizer charitable subtraction to income after the Social Security subtraction and the $19,300 standard deduction, which would yield $19,169 of taxable income." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It applied the full $27,200 Social Security subtraction but omitted the $4,977.18 non-itemizer charitable subtraction and used a $19,000 standard deduction instead of $19,300. That left $24,446 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied 2.85%/4.80% brackets, which are not Minnesota rates; Minnesota's first bracket is 5.35%. It also chose $14,926 of itemized deductions over a standard deduction that it understated as $14,600 by omitting the aged and blind additions. The correct $19,300 standard deduction plus the $4,977 charitable subtraction leave $19,169 taxable." +us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It misstated taxable Social Security as $23,340 (it is $27,200), although its Minnesota AGI of $43,446 landed correctly. It used an $18,560 standard deduction instead of $19,300 and omitted the $4,977.18 charitable subtraction, leaving $24,886 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that standard and senior/disabled subtractions eliminate taxable income. The Minnesota elderly/disabled subtraction is barred at $70,646 AGI, and after the Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 remains taxable at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed the tax is non-positive without computing taxable income. Minnesota taxable income is $43,446 (after the Social Security subtraction) minus the $19,300 standard deduction and the $4,977.18 charitable subtraction, which is $19,169 and yields $1,025.54 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, but the engine adds nothing for it. This raised taxable income above the correct $19,169 and produced $1,094 instead of $1,025.54." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its ~$49,900 of taxable income shows it did not subtract the full $27,200 of taxable Social Security, which Minnesota fully subtracts at this AGI. It also omitted the charitable subtraction, pushing income into the 6.8% bracket instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It added back the $2,013 of tax-exempt interest as a Minnesota addition, which the engine does not do. That raised taxable income to ~$21,055 instead of $19,169 and tax to $1,126 instead of $1,025.54." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $2,640 implies roughly $45,000 of taxable income in the 6.8% bracket, which reflects failing to subtract the full $27,200 of taxable Social Security and the $4,977 charitable subtraction. With both applied, Minnesota taxable income is $19,169, taxed at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used a $19,400 standard deduction instead of $19,300, and a $5,154 charitable subtraction (a plain 50% of contributions over $500) instead of the engine's $4,977.18. Taxable income came out $277 too low ($18,892 vs $19,169)." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the Social Security subtraction and an aged/blind standard deduction but omitted the $4,977.18 non-itemizer charitable subtraction. That left ~$23,265 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $18,342 of taxable income means it subtracted about $25,100 in standard deduction plus charitable subtraction from $43,446. The correct total is $24,277.18 ($19,300 standard deduction plus $4,977.18 charitable), so it overstated deductions by about $827." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It applied the full $27,200 Social Security subtraction but omitted the $4,977.18 non-itemizer charitable subtraction and used a $19,400 standard deduction instead of $19,300. That left $24,046 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted that age-based exemptions eliminate taxable income. Minnesota has no age exemption beyond the standard-deduction additions, and $19,169 remains taxable after the Social Security subtraction, the $19,300 standard deduction and the charitable subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $19,600 standard deduction instead of $19,300 and omitted the $4,977.18 non-itemizer charitable subtraction. That left $23,844 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied the Social Security subtraction and a standard deduction but did not include the $4,977.18 non-itemizer charitable subtraction. That left ~$22,000 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which the engine does not apply. It also used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18, ending with $20,905 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It made no Minnesota subtraction for the $27,200 of taxable Social Security, which is fully subtracted at this AGI. It also itemized $17,023 instead of taking the $19,300 aged/blind standard deduction plus the $4,977 charitable subtraction, taxing $53,623 instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It treated taxable income as near $50,000, meaning it skipped the full $27,200 Social Security subtraction and the charitable subtraction. Applying both plus the $19,300 standard deduction gives $19,169, taxed entirely at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no answer to score against $1,025.54." +us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a ~$5,330 personal exemption, but Minnesota allows exemptions only for dependents, not the filer. It also omitted the $4,977.18 non-itemizer charitable subtraction. Its $18,690 is near the correct $19,169 only because the two errors largely offset." +us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It invoked a 'senior age-65 subtraction' to reach zero taxable income. Minnesota's elderly/disabled subtraction is barred at $70,646 AGI, and after the Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 remains taxable." +us,scenario_122,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It applied the full Social Security subtraction but used a ~$17,950 aged/blind standard deduction instead of $19,300 and omitted the $4,977.18 non-itemizer charitable subtraction. That left $25,496 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It compared $17,367 of itemized deductions to a $15,300 standard deduction that omits the aged and blind additions; the correct standard deduction is $19,300. It also ignored that non-itemizers get the $4,977 charitable subtraction, so the standard route ($24,277 in total) beats itemizing and leaves $19,169 taxable." +us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It never subtracted the $27,200 of taxable Social Security, added back tax-exempt interest and pension, and used a $14,600 standard deduction without the aged/blind additions. It then treated the property tax refund as reducing income tax, although that refund is a separate program and not a nonrefundable credit." +us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a wrong federal AGI ($52,092) and a fabricated $30,100 standard deduction. It then taxed $21,992 at an effective ~6.6%, although that amount lies entirely in Minnesota's 5.35% bracket. The correct base is $19,169 at 5.35%." us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It computed household income correctly at about $77,500 and even said that was near the income limit. It still applied a 2.4% threshold with a 50% state copayment to $4,800 of rent-as-tax and got about $1,470. It then raised that to $2,265 while saying the phaseout reduces the refund. At this household income the 2026 MN renter's credit is $0, so no refundable state credit applies." us,scenario_122,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It found the right household income ($77,459) and rent-as-tax ($4,800). It then applied a mid-schedule bracket: a 2.5% income threshold with a 50% state share, and no maximum-refund cap. The MN renter's credit schedule is fully phased out at this income, so the credit is $0, not $1,431.74." us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It assumed the head qualifies for the Minnesota renter's refund and simply entered a $2,000 'maximum allowable refund'. It never checked the household income of about $77,460 against the renter's credit income schedule. That schedule pays nothing at this income, so state refundable credits are $0." @@ -10059,52 +10190,53 @@ us,scenario_123,child1_head_start_eligible,glm-5.2,parse_contract_failure,missin us,scenario_123,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It left out the 2026 OBBBA $2,000 MFJ non-itemizer charitable deduction, so it taxed $67,802 instead of $65,802. It also added the child's separate $3,220 tax to the parents' tax. The $5,860 it submitted does not match either its $5,440 parents-only figure or its $8,660 combined figure." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It put Child 1's $45,000 of wages into the parents' joint AGI ($145,002), but the child's wages belong on the child's own return. The correct AGI is $100,002 and taxable income is $65,802. Its deduction, bracket and CTC steps were otherwise right." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It made up about $25,200 of mortgage interest from the loan balance and chose to itemize, but no interest paid was listed, so it counts as $0. It also used a $29,200 standard deduction, counted two children for the CTC, and submitted $8,316 without a supporting calculation. The correct path is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, then one $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,000 of combined wages, including the child's $45,000, on the joint return. It also left out the $2,000 non-itemizer charitable deduction and the $2,200 CTC. Correctly, the parents have $65,802 of taxable income and $7,400.24 of tax, reduced by the $2,200 credit." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It used the 2025 standard deduction of $31,500 instead of the 2026 figure of $32,200, and a $2,000 CTC instead of the OBBBA $2,200. It also left out the $2,000 non-itemizer charitable deduction. It then submitted $5,316, which does not match its own $5,724 result." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It made up $29,400 of mortgage interest by applying 7% to the $420,000 balance, but no interest paid was listed, so it counts as $0. It also wrongly denied the CTC on support-test grounds, and its final $12,053 has no supporting calculation. Correctly, the standard deduction plus the $2,000 charitable deduction applies, and the $2,200 CTC is allowed." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It used the correct $100,002 AGI and $32,200 standard deduction but left out the 2026 $2,000 non-itemizer charitable deduction, so it taxed $67,802 instead of $65,802. On top of that, it understated tax on its own base ($7,610 instead of $7,640.24)." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI ($145,002), when they belong on the child's own return. With AGI of $100,002, taxable income is $65,802 and tax is $7,400.24 before the $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It estimated the 2026 parameters by indexing (a $30,750 standard deduction and a 10% bracket ending at $24,450), when OBBBA sets them at $32,200 and $24,800. It also used a $2,000 CTC instead of $2,200 and left out the $2,000 non-itemizer charitable deduction." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted $18,557 of itemized deductions after stating they were less than the $32,200 standard deduction, so it should have taken the standard deduction plus the $2,000 non-itemizer charitable deduction. Its submitted $15,300 also contradicts its own $7,300 estimate." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified overtime deduction for Head, but Head is not flagged as paid hourly, so there is no qualifying overtime premium. It also added the child's separate $3,220 tax. Without those two errors, its parents' figure would have been $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into the joint AGI ($145,002). It also used a $30,800 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and used a $2,000 CTC instead of $2,200." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired for 2026 and applied personal exemptions, 15% brackets and a $1,000 CTC, when OBBBA made the TCJA structure permanent with a $32,200 standard deduction and a $2,200 CTC. It also added the child's separate-return tax to the parents' liability." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA rules for 2026 (personal exemptions of $5,300, itemizing, and a $1,000 CTC), but OBBBA keeps the $32,200 standard deduction, no exemptions, and a $2,200 CTC. It also added the child's $4,896.25 separate tax to the parents' figure." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired: it applied $15,900 of personal exemptions, itemized, used the 15% bracket, and gave a $1,000 CTC. It also added the child's separate $4,855 tax. Under 2026 OBBBA law, the parents alone have $65,802 of taxable income and owe $5,200.24 after the $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired, made up $16,800 of mortgage interest even though no interest paid was listed, and claimed personal exemptions with a $1,000 CTC. It then added the child's separate $4,955 tax. The correct path is the 2026 standard deduction plus the $2,000 charitable deduction, the 10%/12% brackets, and a $2,200 CTC for the parents only." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It started from a joint AGI of $145,002 that includes the child's $45,000 of wages, and it gave no bracket or credit calculation to support $17,290. The parents' AGI is $100,002, and the $2,200 CTC reduces their $7,400.24 of tax." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA 2026 rules (personal exemptions, itemizing, and a $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added the child's separate $4,925 tax to the joint return's liability." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired, so it itemized $18,557, claimed $15,150 of exemptions, and applied a $1,000 CTC. It also added the child's $5,023 separate tax. OBBBA instead gives a $32,200 standard deduction, a $2,000 non-itemizer charitable deduction and a $2,200 CTC, applied to the parents' return only." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. The correct calculation is $100,002 AGI minus $32,200 and $2,000 equals $65,802 of taxable income, $7,400.24 of tax at 10%/12%, and $5,200.24 after the $2,200 CTC. Its $5,304 is $103.76 too high, which fits approximate brackets or deductions rather than the exact 2026 OBBBA figures." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in a $145,002 joint AGI and applied pre-TCJA personal exemptions. It also treated the CTC as phased out, but the 2026 phaseout starts at $400,000 MFJ, so the full $2,200 credit applies." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used a $145,002 joint AGI that includes the child's wages, then itemized $18,557 and claimed pre-TCJA personal exemptions. Under 2026 law the parents' AGI is $100,002, the $32,200 standard deduction plus the $2,000 charitable deduction applies, and the $2,200 CTC is subtracted." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI of $145,002. It also used the 2025 $30,000 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and used a $2,000 CTC instead of $2,200." -us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It added the child's separate $3,312 tax to the parents' liability, but the output covers only the joint return. On the parents' side, it estimated a $30,750 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and used a $2,000 CTC instead of $2,200." -us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's wages into the joint AGI and subtracted a second $16,100 standard deduction for the child on the parents' return. It also claimed the CTC was fully refundable, but the CTC first offsets tax as a nonrefundable credit. Because the $7,400.24 liability is larger than the credit, the whole $2,200 is subtracted here." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It combined Head's and the child's wages into a $145,000+ joint income, so its taxable income was about $116,751 instead of $65,802. The $7,313 it submitted also does not follow from its own taxable income figure." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. Its $14,677 fits taxing the child's $45,000 of wages on the joint return without the full deductions and CTC. Correctly, the parents' $100,002 AGI leaves $65,802 of taxable income and $5,200.24 of tax after the $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the child's $45,000 of wages as part of the joint AGI ($145,002), and it also left out the 2026 $2,000 non-itemizer charitable deduction, so it taxed $112,802 instead of $65,802." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI ($145,002), when they are taxed on the child's own return. Its deductions and $2,200 CTC were otherwise correct, so it overstated taxable income by $45,000." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the child's wages on the joint return and used the outdated $30,000 standard deduction instead of the 2026 $32,200. It also left out the $2,000 non-itemizer charitable deduction, which produced $15,128.44 of regular tax instead of $7,400.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the parents' joint tax as $5,200.24, then added the child's separate $3,220 tax. That is wrong because this output covers only the parents' tax unit." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It added the child's separate $3,220 tax to the parents' liability. It also left out the $2,000 non-itemizer charitable deduction, so the parents' taxable income was $67,802 instead of $65,802." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified overtime deduction for Head, but Head is not flagged as paid hourly, so there is no qualifying overtime. It also left out the $2,000 non-itemizer charitable deduction and added the child's separate $3,220 tax." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly arrived at $5,200.24 for the joint return, then added the child's separate $3,220 tax. That is wrong because this output covers only the parents' tax unit." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint income and deducted mortgage interest even though no interest paid was listed. The correct calculation uses AGI of $100,002 and the $32,200 standard deduction plus the $2,000 charitable deduction, then subtracts the $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied post-TCJA-sunset law for 2026 (restored exemptions, 15% bracket, and a $1,000 CTC) and deducted PA income tax as part of itemizing. It also added the child's separate $4,851 tax. Under OBBBA, the standard deduction plus the $2,000 charitable deduction applies, the brackets are 10%/12%, and the CTC is $2,200, on the parents' return only." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired: it used exemptions, 10/15/25% brackets and an unindexed $110,000 CTC phaseout that wiped out the credit. It also put the child's wages in the joint AGI. Under 2026 law the CTC is $2,200 with a $400,000 phaseout, and the parents' AGI is $100,002." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA 2026 law ($5,400 exemptions, 15%/25% brackets, and a CTC phaseout at $110,000) and included the child's $45,000 of wages in the joint AGI. Under OBBBA, the parents' $65,802 of taxable income is taxed at 10%/12%, and the full $2,200 CTC is subtracted." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA rules (three $5,434 exemptions, itemizing, the 15% bracket, a 10% medical floor, and a $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added the child's $4,846 separate tax." -us,scenario_123,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It reached roughly the right $65,802 of taxable income and about $5,210 of tax for the parents, then added the child's separate return tax (about $3,225). That child tax is not part of the joint tax unit's federal income tax." -us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." -us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $12,000 qualified overtime deduction for Head, who is not flagged as paid hourly. It also included the child's $45,000 of wages in the joint AGI and left out the $2,000 non-itemizer charitable deduction, so it taxed $100,802 instead of $65,802." -us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed credits wipe out the liability, but the parents' $7,400.24 of tax on $65,802 of taxable income far exceeds the $2,200 nonrefundable CTC. It also wrongly brought refundable credits into an output that is measured before refundable credits." -us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into the joint AGI and took a $12,500 overtime deduction for Head, who is not flagged as paid hourly. It also left out the $2,000 non-itemizer charitable deduction, so taxable income came out at $100,302 instead of $65,802." -us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's wages in a $145,002 AGI, made up $27,300 of mortgage interest from the balance, and used 2025 brackets with a $2,000 CTC. The correct calculation uses $100,002 of AGI, the $32,200 standard deduction plus the $2,000 charitable deduction, 2026 brackets, and a $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It made up deductions and credits that do not apply: a QBI deduction on W-2 wages, mortgage interest that was not listed, a CDCC with no care expenses, and the foreign earned income exclusion. Its own math still left $5,687 of tax, yet it submitted 0. The correct result is $7,400.24 less the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It left out the 2026 $2,000 non-itemizer charitable deduction, so it got taxable income of $67,802 and $5,440 after the CTC instead of $65,802 and $5,200.24. It then added a separate $3,220 child return and submitted $5,860, which matches neither of its own computations." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It put the dependent child's $45,000 wages into the joint return's AGI ($145,002). The parents' AGI is $100,002. The rest of its method (standard deduction $32,200 + $2,000 non-itemizer deduction, $2,200 CTC) was right, so the extra $45,000 taxed at 12%/22% accounts for the whole $6,400 overstatement." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It used an outdated $29,200 standard deduction and made up about $25,200 of mortgage interest from the loan balance, even though no interest paid was listed. It also counted two CTC-qualifying children when there is only one, and its final $8,316 does not follow from any of its stated steps." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the combined $145,000 of wages, including the dependent child's $45,000, on the joint return. It also left out the $2,000 non-itemizer charitable deduction and the $2,200 nonrefundable CTC for the 16-year-old, which together produced a rough $14,500." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It used the 2025 $31,500 standard deduction instead of the 2026 $32,200, left out the $2,000 non-itemizer charitable deduction, and applied a $2,000 CTC instead of $2,200. It then cut its own $5,724 to $5,316 with an unexplained bracket adjustment." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It made up $29,400 of mortgage interest from the $420,000 balance, although no interest paid was given (it counts as $0), and chose to itemize because of it. It also wrongly denied the $2,200 CTC on a support-test theory, and its $12,053 does not follow from its own $5,765 computation." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It left out the 2026 $2,000 non-itemizer charitable deduction, so taxable income was $67,802 instead of $65,802. It also miscalculated the bracket tax on that amount ($7,610 instead of $7,640.24), which gave $5,410 after the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 wages in the joint AGI ($145,002 instead of $100,002), which raised taxable income to $110,802. The deductions and the $2,200 CTC were otherwise right." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It estimated the 2026 parameters by indexing 2025 figures instead of using the OBBBA values: a $30,750 standard deduction instead of $32,200, a 10% bracket ending at $24,450 instead of $24,800, and a $2,000 CTC instead of $2,200. It also left out the $2,000 non-itemizer charitable deduction, which gave $5,821." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It wrongly said that $18,557 of itemized deductions beat the $32,200 standard deduction, so it subtracted only $18,557 and left out the $2,000 non-itemizer charitable deduction. It then submitted $15,300, which contradicts its own estimate of about $7,300." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It gave the head a $13,650 qualified-overtime deduction even though the head is not flagged as paid hourly, so no overtime premium is computed. It then added a separate $3,220 child return, although the output covers only the joint unit, whose tax is $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 wages in the joint AGI. It also used a $30,800 standard deduction instead of $32,200, left out the $2,000 non-itemizer charitable deduction, and applied a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA sunset law (personal exemptions, a 15% bracket, a $1,000 CTC) instead of the 2026 OBBBA rules ($32,200 standard deduction plus $2,000 non-itemizer deduction, 10%/12% brackets, $2,200 CTC). It also added a separate child return to the joint unit's tax." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA sunset rules (three $5,300 personal exemptions, a 15% bracket, a $1,000 CTC) instead of 2026 OBBBA law. It also added a $4,896.25 child return to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA sunset law (personal exemptions, itemizing $18,557 against a small restored standard deduction, a $1,000 CTC) instead of the 2026 $32,200 standard deduction, $2,000 non-itemizer deduction and $2,200 CTC. It also added the child's separate $4,855 tax." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA had sunset, so it used personal exemptions, 15% brackets and a $1,000 CTC. It also made up $16,800 of mortgage interest from the balance and added the child's separate $4,955 return to the joint unit's tax." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It started from a $145,002 joint AGI that includes the dependent child's $45,000 wages. It also did not apply the $32,200 standard deduction, the $2,000 non-itemizer deduction or the $2,200 CTC to the parents' $100,002 AGI, which produced $17,290." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA 2026 rules (personal exemptions, itemizing, a $1,000 CTC) instead of the OBBBA $32,200 standard deduction, $2,000 non-itemizer charitable deduction and $2,200 CTC. It then added the child's separate $4,925 tax." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the joint return under expired pre-TCJA law (three exemptions, 15% bracket, $1,000 CTC), which gave $7,784 instead of $5,200.24. It then added a $5,023 child return that does not belong in the joint unit's output." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"With no derivation shown, its $5,304 is $103.76 above the correct $5,200.24 ($65,802 taxable income at 10%/12% gives $7,400.24, minus the $2,200 CTC). That gap equals about $865 of extra taxable income at 12%, which fits approximated deductions or bracket thresholds instead of the exact 2026 $32,200 + $2,000 deductions and $24,800 10% bracket." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed a $145,002 AGI that includes the dependent child's wages. It also applied pre-TCJA personal exemptions and itemizing, and wrongly phased out the CTC, although the $2,200 CTC is fully available at a $100,002 AGI." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 wages into a $145,002 joint AGI and used pre-TCJA itemizing with personal exemptions. The joint AGI is $100,002, with a $32,200 standard deduction, a $2,000 non-itemizer deduction and a $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 wages in the joint AGI ($145,002). It also used the 2025 $30,000 standard deduction instead of $32,200, left out the $2,000 non-itemizer deduction, and used a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It added the child's separate $3,312.08 tax to the joint unit's liability. On the joint return it guessed a $30,750 standard deduction and a $24,293 10% bracket, left out the $2,000 non-itemizer deduction, and used a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the CTC as fully refundable and subtracted none of it, although the $2,200 CTC is used nonrefundably against $7,400.24 of liability. It also included the child's $45,000 wages in the joint AGI and subtracted an extra $16,100 dependent standard deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It put the dependent child's wages into the joint income, which gave about $116,751 of taxable income instead of $65,802. Its $7,313 also does not match the bracket tax on its own taxable income." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"With no derivation given, its $14,677 is far above the $5,200.24 that results from taxing $65,802 and subtracting the $2,200 CTC. That level only arises from adding the dependent child's $45,000 wages to the joint return's income." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It used one joint AGI of $145,002 that includes the dependent child's $45,000 wages, although the parents' AGI is $100,002. It also left out the $2,000 non-itemizer charitable deduction, which gave $112,802 of taxable income." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 wages in the joint AGI ($145,002 instead of $100,002), which gave $110,802 of taxable income instead of $65,802. The deductions and the $2,200 CTC were applied correctly." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"Its $15,128.44 pre-credit tax implies $115,002 of taxable income. That means it put the child's $45,000 wages into the joint AGI, used the 2025 $30,000 standard deduction instead of $32,200, and left out the $2,000 non-itemizer deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed $5,200.24 for the joint unit, then added a separate $3,220 tax on the child's wages. The output is the joint tax unit's liability, so no child return belongs in it." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate $3,220 child return to the joint unit's tax. It also left out the $2,000 non-itemizer charitable deduction, which gave $67,802 of taxable income and $7,640.24 of tax instead of $65,802 and $7,400.24." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified-overtime deduction for the head, who is not flagged as paid hourly, and left out the $2,000 non-itemizer charitable deduction. It also added the child's separate $3,220 tax to the joint unit's output." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint unit's $5,200.24 right, then added the dependent child's separate $3,220 wage tax. The requested output is only the joint tax unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It combined the head's and the child's wages into one $145,002 AGI and claimed a mortgage interest deduction, although no interest paid was listed. The joint AGI is $100,002, and the $32,200 standard deduction plus the $2,000 non-itemizer deduction apply." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 2026 TCJA-sunset rules (personal exemptions, restored 15% bracket, $1,000 CTC) instead of the OBBBA-permanent $32,200 standard deduction, $2,000 non-itemizer deduction and $2,200 CTC. It also added the child's separate $4,851 tax." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA had sunset (personal exemptions, 15%/25% brackets, CTC phased out above $110,000) and put the child's $45,000 wages into a $145,002 joint AGI. Under 2026 law the parents' $100,002 AGI keeps the full $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA sunset law with three $5,400 exemptions, a 15% bracket and a CTC phased out above $110,000, and it included the child's wages in the AGI. The correct base is a $100,002 AGI with the 2026 $32,200 standard deduction, the $2,000 non-itemizer deduction and the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the joint return under expired pre-TCJA law (itemizing $18,557 with three exemptions, 15% bracket, $1,000 CTC) instead of the 2026 $32,200 + $2,000 deductions and $2,200 CTC. It also added a $4,846 child return." +us,scenario_123,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It reached the correct $65,802 taxable income for the joint unit, but then added a separate child tax of about $3,225 to the joint unit's output. It also rounded the joint tax to about $7,410 instead of $7,400.24." +us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It made up a $12,000 qualified-overtime deduction for the head, who is not flagged as paid hourly, and included the dependent child's $45,000 wages in the joint AGI. It also left out the $2,000 non-itemizer charitable deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It counted refundable credits against liability and said the tax rounds to $0. The joint unit has $7,400.24 of pre-credit tax and only a $2,200 nonrefundable CTC, which leaves $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 wages in the joint AGI and took a $13,650 overtime-premium deduction for the head, who is not flagged as paid hourly. It also used a $31,500 standard deduction instead of $32,200." +us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 wages into a $145,002 joint AGI and took a $12,500 overtime deduction for the head, who is not flagged as paid hourly. It also left out the $2,000 non-itemizer charitable deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It made up $27,300 of mortgage interest from the $420,000 balance, although no interest paid was given, so it itemized $45,857. It also included the child's wages in the AGI, used 2025 brackets and a $30,000 standard deduction, and applied a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It made up deductions and credits that do not apply: a QBI deduction on W-2 wages, imputed mortgage interest, a CDCC with no care expenses, and a foreign earned income exclusion. It then reported $0, even though its own figures leave $5,687 of tax after the CTC." us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model paid out a flat $1,600 ""refundable CTC"" without running the ACTC limitation: the Child Tax Credit is first applied as a nonrefundable credit, and with MFJ taxable income of roughly $112,800 (AGI $145,002 less the 2026 MFJ standard deduction) generating about $14,000 of federal income tax, the entire CTC is absorbed nonrefundably and no additional child tax credit arises. It compounded this by inventing $205 of EITC at $145,002 of AGI, when the one-child EITC for joint filers is fully phased out in the upper $50,000s." us,scenario_123,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model asserted that satisfying the 15%-of-earnings phase-in makes ""the full credit refundable,"" skipping the step that governs here: the CTC offsets the household's roughly $14,000 federal income tax liability as a nonrefundable credit first, and only the unused remainder — zero — flows to the refundable additional child tax credit, which is separately capped at $1,700 per child in 2026 rather than the full $2,400 the model reported." us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model claimed ""no limitation from the nonrefundable portion,"" which is exactly backwards: the nonrefundable CTC is applied first against the household's roughly $14,000 of federal income tax before refundable credits, exhausting the credit and leaving $0 of additional child tax credit. Its $1,500 figure is also a stale pre-2022 ACTC cap rather than the 2026 $1,700 ceiling." @@ -10163,52 +10295,53 @@ us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_ us,scenario_123,spouse_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the spouse's MAGI was below Pennsylvania's adult Medicaid threshold. The spouse's MAGI is 5.31 times FPL, far above the applicable income limits, and the spouse qualifies through no other Medicaid category." us,scenario_123,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed the parents' joint PA tax of $3,070.06. It then added $1,381.50 for a separate child return, even though the prompt puts the child in the parents' single tax unit as a dependent whose wages are not taxed on that return. It also submitted $4,453.06, which does not match its own sum of $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002 by pooling the dependent child's $45,000 of wages into the tax unit's PA base. Only the head's and spouse's $100,002 is PA taxable income for this unit, so the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model invented PA standard deductions of $15,000 and $8,500, but PA allows no standard deduction or personal exemption. It also included the dependent child's $45,000 of wages. The correct base is the parents' $100,002 at 3.07%, which is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model used a base of roughly $145,000 that includes the dependent child's wages. Its $5,900 also does not match 3.07% of its own base, which is $4,451.56. The tax unit's PA base is only the parents' $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the parents' compensation, giving a $145,002 base. The dependent's wages are not part of the joint tax unit's PA income. It also submitted $4,452.08 instead of its own computed $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model included Child 1's $45,000 of wages in a combined $145,002 base. The tax unit's PA taxable income is only the head's and spouse's $100,002, so the tax is $3,070.06. It also submitted $4,452.61, which differs from its own computed $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly found about $3,070 for the head and spouse. It then added $1,381.50 of separate tax on the child's wages, even though the child is a dependent in the same single tax unit and no separate child return exists." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002 by counting the dependent child's $45,000 of wages as tax-unit income. The PA base is only the parents' $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly worked out that the joint return covers $100,002 and owes $3,070.06. It then overrode that result and added the child's $45,000 because 'all are listed in one tax unit.' Being in one tax unit is exactly why the dependent's wages are excluded from the parents' return." -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly limited the base to the parents' $100,002 and computed $3,070.06. It then arbitrarily reduced this to $3,060, citing nonexistent 'compensation-only interest treatment nuances.' PA taxes interest at the same flat 3.07%." -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed $3,070.06 for the parents. It then added a 'separate unit' tax of $1,381.50 for the child, but the prompt places the child in the same single tax unit as a dependent, so no child return is taxed." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 by including the dependent child's $45,000 of wages. The tax unit's PA income is the parents' $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added $1,381.50 of tax on the dependent child's wages, which are not part of the parents' tax unit. It also dropped the couple's $2 of taxable interest. The correct figure is 3.07% × $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002 by pooling the dependent child's wages into the tax unit's base. Only the head's and spouse's $100,002 is taxed, which gives $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted the $16,100 of charitable donations, but PA allows no charitable or itemized deductions. It also added the dependent child's $45,000 of wages. The correct base is the parents' $100,002 with no deductions, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model summed $100,001, $1, and the child's $45,000 into a $145,002 base. The dependent child's wages are not income of the parents' tax unit, so the base is $100,002 and the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model used a 3% rate instead of PA's 3.07% flat rate. It applied that rate to $145,000, which includes the dependent child's wages. The correct calculation is 3.07% × the parents' $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated $145,002 of 'household' wages and interest as the PA base, which pulls in the dependent child's $45,000. The tax unit's base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed about $3,070 on the parents' joint $100,002. It then added $1,382 for a child return that does not exist, because the child is a dependent inside the single tax unit." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation, and its $3,385 implies a base of about $110,260 at 3.07%, which matches no income figure in the household. The correct calculation taxes only the parents' $100,002 of compensation and interest at 3.07%, which is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to a $145,002 'household' total that includes the dependent child's wages. Only the parents' $100,002 is the tax unit's PA income, so the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model used $145,002 as taxable income by counting the dependent child's $45,000 of wages. The joint tax unit's PA base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model counted $145,000 of compensation, which includes the dependent child's wages that are not on the parents' return. The PA base is $100,000 of wages plus $2 of interest, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly computed $3,070.06 for the joint return. It then added $1,381.50 for a separate child filing, but the prompt places the child in the single tax unit as a dependent, so no separate child tax is computed." -us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 by adding the dependent child's $45,000 of wages to the parents' income. The tax unit's PA base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed household-wide wages, including the dependent child's, and its $4,807 matches no correct product of 3.07% and any listed income total. The correct figure is 3.07% × the parents' $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model ran a federal-style itemized-deduction computation, but PA has no itemized deductions and taxes income at a flat 3.07%. Its $9,055 is nearly triple the correct $3,070.06, which is the tax on the parents' $100,002." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent child's $45,000 of wages in a $145,002 base. The joint tax unit's PA income is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent child's wages in a $145,002 base. It also misstated the product as $4,455.06 when the correct product is $4,451.56. The correct tax is 3.07% × $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied 3.07% to $145,002, which includes the child's $45,000 of wages. A dependent's wages are not income of the parents' tax unit, so the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 of 'household' wages and interest, pulling in the dependent child's earnings. The tax unit's PA base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,002 of household wages and interest, but the dependent child's $45,000 belongs outside the joint tax unit's PA income. The correct calculation is 3.07% × $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly taxed the parents' $100,002. It then added a separate tax on the child's $45,000 of wages, even though the prompt puts the child in the single tax unit as a dependent with no separate return." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed $145,000 of wages, which includes the dependent child's $45,000, which is not on the parents' return. The correct base is $100,000 of wages plus $2 of interest, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model included the dependent child's $45,000 of wages in the PA base. Only the head's and spouse's $100,002 is taxed for this tax unit, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed a combined base of about $145,000 that includes the dependent child's wages. The joint tax unit's PA income is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed combined compensation of $145,002 by counting the dependent child's wages. Only the parents' $100,002 is the tax unit's PA base, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly noted that PA has no deductions. It then set taxable income at $145,002 by including the dependent child's wages, which are not income of the parents' tax unit. The correct tax is 3.07% × $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model used $145,002 as taxable income, which includes the child's $45,000 of wages. The dependent's earnings are excluded from the joint tax unit, so the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the head's and spouse's income. The PA base for the single tax unit is only the parents' $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed 'both filing units,' treating the child as a separate filer. The prompt defines one tax unit in which the child is a dependent and the child's wages are not taxed, so the answer is the parents' $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no reasoning, and its $2,939.86 implies a base of about $95,760, roughly $4,242 below the parents' $100,002. That shortfall shows it subtracted a deduction or exclusion that PA does not allow. The correct tax is 3.07% × $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly applied no deductions but used $145,002 as taxable income by including the dependent child's wages. The tax unit's base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model acknowledged that PA liability is positive but submitted 0, deferring to an imagined structured output. It also wrongly cited a PA standard deduction. The correct calculation is the flat 3.07% on the parents' $100,002, which is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly allowed no deductions but taxed $145,002 by including the dependent child's $45,000 of wages. The joint tax unit's base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the parents' compensation, giving a $145,002 base, and also misrounded the result to $4,451.57. The dependent's wages are excluded from the tax unit, so the tax is 3.07% × $100,002 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model used federal taxable income of $67,525 as the PA base, but PA does not start from federal taxable income. PA taxes gross compensation and interest with no federal deductions. The model also submitted $2,565.30, which contradicts its own $2,073.02. The correct base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly got $3,070.06 for the parents' return. It then added a separate $1,381.50 PA return for the dependent child, but the child's $45,000 of wages is outside the head/spouse tax-unit base. It also submitted $4,453.06, which doesn't match its own $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which wrongly includes the dependent child's $45,000 of wages. The tax unit's PA base is only the head and spouse's $100,002, so the tax is $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It made up $15,000 and $8,500 PA standard deductions, but PA has no standard deduction. It also added the dependent child's $45,000 of wages to the base. The correct base is the head and spouse's $100,002 with no deductions, which gives $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It used a base of about $145,000, which includes the dependent child's wages. Even then, $5,900 implies a rate of about 4.07% instead of PA's flat 3.07%. The correct result is 3.07% × the head and spouse's $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages as PA-taxable for the tax unit, giving a $145,002 base instead of the head and spouse's $100,002. It also submitted $4,452.08, which doesn't match its own $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It added Child 1's $45,000 of wages to the head and spouse's income for a $145,002 base. The dependent's wages don't belong in the tax unit's PA base, which is $100,002 and gives $3,070.06. It also submitted $4,452.61 against its own $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly got about $3,070 for the head and spouse. It then added $1,381.50 for the child as a separate filer, but the dependent child is part of the single tax unit and its wages are excluded from that unit's PA base." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which counts the dependent child's $45,000 of wages. The tax unit's PA taxable income is the head and spouse's $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly found that the joint return gives $100,002 × 3.07% = $3,070.06. It then reversed itself and added the child's $45,000 because everyone is in one tax unit. A dependent's wages are not part of the head/spouse PA base." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly found the base of $100,002 and the tax of $3,070.06. It then cut the answer to $3,060 citing 'interest treatment nuances,' but PA has no such adjustment, so the tax is exactly $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the parents' $3,070.06. It then added a separate child unit owing $1,381.50, but the child is a dependent inside the single tax unit and that unit's PA base excludes the child's wages." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 as the base, which includes the dependent child's $45,000 of wages. The tax unit's PA base is only the head and spouse's $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It added $1,381.50 of tax on the dependent child's $45,000 of wages, which is excluded from the tax unit's PA base. It also left out the $2 of taxable interest. The correct base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's $45,000 of wages. The PA base for the head/spouse tax unit is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $16,100 charitable donation, but PA allows no charitable deduction. It also added the dependent child's $45,000 of wages. The correct base is the head and spouse's $100,002 with nothing subtracted." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It counted Child 1's $45,000 of wages in the tax unit's PA taxable income, giving $145,002. The dependent's wages are excluded, so the base is $100,002 and the tax is $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used a 3% rate instead of PA's 3.07%. It also applied that rate to $145,000, which includes the dependent child's wages. The correct calculation is 3.07% × $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to a household total of $145,002 that includes the dependent child's wages. The tax unit's PA base is the head and spouse's $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed about $3,070 for the joint filers. It then added about $1,382 for Child 1 as a separate filer, but the dependent's wages are not part of the tax unit's PA tax." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. $3,385 implies a 3.07% base of about $110,260, which doesn't match any valid PA income class. The correct base is the head and spouse's $100,000 of wages plus $2 of interest, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used a household total of $145,002 that includes the dependent child's $45,000 of wages. The tax unit's PA base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002, which wrongly includes the dependent child's wages. The head/spouse tax unit's PA taxable income is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted $145,000 of compensation, including Child 1's $45,000. Only the head's $100,000 of wages and the $2 of interest belong in the tax unit's PA base, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed $3,070.06 for the joint return. It then added $1,381.50 for the child filing separately, but the dependent child is part of the single tax unit and its wages are excluded from that unit's PA tax." +us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It summed $100,000 + $45,000 + $2, which wrongly includes the dependent child's wages. The PA base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"At 3.07%, its $4,807 implies a base of about $156,580. That is more than all household income and includes the dependent child's wages. The correct base is the head and spouse's $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It ran a federal-style calculation using itemized medical and real-estate-tax deductions, but PA allows neither. Its $9,055 is far above 3.07% of any base in this household. The correct result is a flat 3.07% × $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included Child 1's $45,000 of wages for a $145,002 base. The dependent's wages are excluded from the tax unit's PA income, which is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which includes the dependent child's wages, and still got the product wrong: it gave $4,455.06 instead of $4,451.56. The correct base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's $45,000 of wages. The tax unit's PA base is only the head and spouse's $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 of household wages and interest. The dependent child's $45,000 of wages is excluded from the tax unit's PA base, which leaves $100,002 and a tax of $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 of household income, which includes the dependent child's wages. The PA base is the head and spouse's $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly taxed the parents' $100,002. It then added tax on the child's $45,000 as a separate return, but the dependent's wages are not part of the single tax unit's PA tax." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,000 of wages, which includes the dependent child's $45,000. Only the head's $100,000 of wages plus $2 of interest belong in the tax unit's PA base." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 of wages in the PA base. The head/spouse tax unit's taxable income is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to about $145,000 of combined income, which includes the dependent child's wages. The tax unit's PA base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 of combined compensation, which counts the dependent child's $45,000 of wages. Those wages are excluded from the head/spouse tax unit's PA base." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly found that PA allows no deductions. However, it used a $145,002 base that includes the dependent child's wages; the correct base is $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It treated $145,002 as the PA taxable income, which includes Child 1's wages. The dependent's wages are not in the tax unit's base of $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly added the child's $45,000 to the head and spouse's income. The tax unit's PA base excludes the dependent's wages and is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the child as a second filing unit and added that tax. The output covers only the single head/spouse tax unit, whose $100,002 base gives $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning. $2,939.86 implies a 3.07% base of about $95,760, about $4,240 below the head and spouse's $100,002, which matches subtracting a deduction PA does not allow. PA taxes the full $100,002 of compensation and interest, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly applied no deductions. However, it used a $145,002 base that includes the dependent child's wages; the tax unit's base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It said PA tax liability is positive and then reported 0 anyway. PA's flat 3.07% applies to the head and spouse's $100,002 with no forgiveness at this income, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed about $145,002, which includes the dependent child's wages. The tax unit's PA base is the head and spouse's $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly excluded the charitable, medical, and property-tax deductions. However, it added the dependent child's $45,000 of wages, making the base $145,002 instead of $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It counted Child 1's $45,000 of wages in the tax unit's PA taxable income. The correct base is the head and spouse's $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It started from a federal taxable income of $67,525. PA does not start from federal taxable income and allows no standard or itemized deductions, so the base is the head and spouse's gross $100,002. It also submitted $2,565.30, which doesn't match its own $2,073.02." us,scenario_123,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv index 3b12cc9d..e3e2f7f8 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv @@ -1,5 +1,5 @@ country,scenario_id,variable,wrong_model_count,case_failure_sources,case_failure_subtypes,reference_suspect,reference_bug_hypothesis,case_annotation -us,scenario_000,federal_income_tax_before_refundable_credits,43,llm_error,household_unit_or_filing_status,False,,"The 'is a surviving spouse' flag does not create qualifying-surviving-spouse status. IRC §2(a) requires the filer to maintain a home for a dependent child, and none is listed, so the head files as single. Under single status, the $25,000/$34,000 thresholds make $18,542.70 of Social Security taxable, so AGI is $50,437.11. The post-OBBBA single standard deduction is $16,100 plus the $2,050 aged add-on, for $18,150, and the $6,000 senior deduction also applies. That leaves $26,287.11 of taxable income, taxed under the 2026 single brackets: 10% up to $12,400 and 12% above it, for $2,906.45. Most wrong answers made one of three errors: they used joint-style QSS thresholds and deductions, assumed the TCJA sunset (personal exemption and a 15% bracket), or left out the $6,000 senior deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,44,llm_error,household_unit_or_filing_status,False,,"The head is a widowed person with no dependent child. Qualifying-surviving-spouse status requires a dependent child, so they file as single. That means taxable Social Security is figured with the $25,000/$34,000 thresholds: $4,500 + 85% × ($50,520 − $34,000) = $18,542.70, giving AGI of $50,437.11. Under OBBBA rules for 2026 (TCJA has not expired), deductions are the $16,100 single standard deduction, the $2,050 aged add-on and the $6,000 senior deduction, for $24,150 in total. Taxable income is $26,287.11, taxed at 10% up to $12,400 and 12% above that, which gives $2,906.45. The wrong answers fall into four groups: using joint/QSS thresholds and deductions, applying pre-TCJA sunset law (personal exemption, 15% bracket, no SALT cap), leaving out the $6,000 senior deduction, or miscomputing taxable Social Security." us,scenario_000,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_000,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Texas Medicaid eligibility at age 77 requires qualification through an aged Medicaid pathway; age alone does not establish a qualifying category. This person has no Medicaid eligibility category, receives no SSI, and has MAGI income of 3.98 times FPL, so the immigration-status requirement does not create eligibility by itself." us,scenario_000,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wage and salary compensation, not to Social Security retirement benefits, pensions, IRA distributions, interest, or veterans benefits. Because the household reports no wages or other employee compensation, the employee Social Security and Medicare tax base is zero and payroll tax is $0." @@ -10,7 +10,7 @@ us,scenario_001,snap,1,llm_error,thresholds_rates,False,,"SNAP first applies the us,scenario_001,spouse_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The spouse is 69, which excludes them from Virginia's ACA expansion adult group (ages 19-64) and from every other MAGI category, and the household's Medicaid MAGI of $44,289 ($29,064 wages plus $15,225 Social Security, counted in full for MAGI) sits at 2.05x the 2-person FPL, far above the 138% FPL expansion ceiling. The only route left for a 65+ adult is the non-MAGI aged/blind/disabled pathway, which is not conferred by age: it requires countable income at or below Virginia's SSI-related ABD standard (roughly 80% FPL, about $17,300 for a couple) with a $3,000 couple resource limit, and $44,289 clears that standard by more than 2.5x even after the $20 general and $65-plus-half earned disregards (countable about $2,427/month). The spouse also received $0 SSI, so the SSI-recipient pathway never triggers, leaving medicaid_category = NONE. Every wrong model treated age 65+ as itself creating an aged pathway, or assumed the spouse's own $0 earnings rather than the couple's combined countable income governs that pathway's income test." us,scenario_001,spouse_medicare_eligible,1,llm_error,categorical_eligibility,False,,"PolicyEngine's Medicare eligibility variable applies the standard age criterion, which both spouses satisfy because they are 71 and 69. The spouse's eligibility therefore equals Yes without requiring a separately listed work-credit history." us,scenario_001,state_income_tax_before_refundable_credits,7,llm_error,state_local_rule,False,,"Virginia gives each taxpayer age 65 or older a $12,000 age deduction under Va. Code § 58.1-322.02(10)-(11), reduced dollar-for-dollar only when adjusted federal AGI — a measure that excludes taxable Social Security — exceeds $75,000 on a joint return. Virginia also subtracts the federally taxable Social Security ($2,338.25 of the $15,225), so Virginia AGI is $29,064 and AFYA is $29,064, far below the $75,000 joint threshold; both spouses therefore take the full $12,000, and $24,000 of age deductions alone wipes out Virginia AGI before the joint standard deduction and the $930 personal plus $800 age exemptions are even applied. Taxable income is $0 and the tax is $0 at any 2026 standard-deduction level. Every wrong answer either replaced the $12,000-per-person age deduction with Virginia's small $800 additional 65+ exemption, granted it to only one spouse, or left taxable Social Security inside Virginia AGI." -us,scenario_002,federal_income_tax_before_refundable_credits,10,prompt_ambiguity,taxable_income_or_deductions,False,,"The correct 2026 AGI is $23,130: pension $18,340 + interest $400 + $4,390 of taxable Social Security. The $24,000 disability benefits and $48,000 veterans benefits are excluded from gross income, so provisional income is $18,740 + half of $44,080 = $40,780, which sits between the MFJ $32,000 and $44,000 thresholds and therefore includes only 50% × ($40,780 − $32,000) = $4,390 of benefits. That AGI is below the 2026 MFJ standard deduction alone ($32,200 base + two $1,650 age-65 additions = $35,500), and further below it once the $6,000-per-senior deduction ($12,000) is applied, so taxable income and tax before refundable credits are $0, and NIIT is $0 because AGI is far under $250,000. Nine models manufactured taxable income by pulling the non-taxable disability benefits into gross/provisional income (which also flips Social Security from the 50% tier to the 85% tier), by taxing 100% of Social Security, or by filing the couple as a single individual. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." +us,scenario_002,federal_income_tax_before_refundable_credits,11,prompt_ambiguity,taxable_income_or_deductions,False,,"PolicyEngine does not count the $24,000 disability benefits or the $48,000 veterans benefits in gross income. That leaves $18,740 of other income: $18,340 of pension plus $400 of interest. Provisional income is $18,740 plus half of Social Security ($22,040), or $40,780, which is below the $44,000 MFJ second-tier threshold. Only 50% of the excess over $32,000 is taxable, so $4,390 of Social Security is taxable and AGI is $23,130. The 2026 MFJ standard deduction with both spouses 65+ is $35,500, before the $6,000-per-senior deduction, so it wipes out all of that income and tax is $0. Most models counted the disability benefits as taxable, made all of Social Security taxable, or both, and then also understated the deductions. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." us,scenario_002,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to covered wages and other employment compensation, not Social Security retirement benefits, pensions, disability benefits, veterans benefits, or taxable interest. Because the household has no wages or other payroll-taxable earnings, every employee-side payroll-tax component is zero." us,scenario_002,spouse_medicaid_eligible,1,llm_error,household_unit_or_filing_status,False,,"Medicaid eligibility for the spouse cannot be established by treating the spouse's lack of individually reported income as zero countable income. The spouse qualifies through no Washington Medicaid pathway: the engine assigns category NONE, records no SSI, and calculates household MAGI at 2.90 times FPL, above the applicable MAGI limits." us,scenario_003,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"To match the reference, a model has to: include 85% of Social Security; net the $1,080 short-term gain against the $2,662 long-term loss for a $1,582 deduction; take no traditional 401(k)/IRA deduction because nobody has compensation; and apply the 2026 rules made permanent by OBBBA. Those rules are a $32,200 joint standard deduction (no age-65 or senior add-on, since the head is 64 and the spouse 61), a 10% bracket to $24,800, a 12% bracket to $100,800 and 22% above that, with $3,760 of qualified dividends taxed at 15%. The largest group of wrong answers assumed the TCJA expired, bringing back personal exemptions and the 10/15/25% brackets. Another group used 2025 or earlier standard deductions and bracket limits. The rest invented deductions (retirement contributions without earnings, car-loan interest, age-65 amounts), double-counted capital gains or losses, or submitted a number that did not match their own working." @@ -24,18 +24,18 @@ us,scenario_004,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax app us,scenario_004,spouse_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"New York's ACA expansion pathway requires the spouse's MAGI income level to be at or below 138% of FPL. The household's applicable MAGI is 2.39 times FPL, and the 51-year-old spouse has no disability, blindness, SSI, age-based, dependent, or other categorical pathway, so the Medicaid category is NONE." us,scenario_004,state_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"Both spouses show 40-hour work weeks but no wage amount, and the prompt directs that unlisted numeric inputs be treated as 0, so the household's only non-Social-Security income is $1,200 of taxable interest less a $135 partnership loss = $1,065. Provisional income is $1,065 + half of $51,475 = $26,803, below the $32,000 married-filing-jointly base, so none of the Social Security benefit is federally taxable, and New York subtracts any taxable Social Security from federal AGI regardless. New York AGI of $1,065 is entirely absorbed by the ~$16,050 MFJ standard deduction, leaving $0 of NY taxable income and $0 of tax before refundable credits. Every wrong answer either taxed the $51,475 Social Security benefit, imputed phantom wages from the hours-worked fact, or both." us,scenario_004,state_refundable_credits,1,llm_error,state_local_rule,False,,"This couple has no wages or rent, so its federal AGI is only about $1,065: $1,200 of interest minus a $135 partnership loss. None of the $51,475 in Social Security is taxable, because provisional income is below the $32,000 joint base. The only NY credit this low income could trigger is the household credit under Tax Law §606(b), and that credit is nonrefundable. It can only reduce NY tax, which is already zero, and it never counts toward state_refundable_credits. None of New York's refundable credits apply either: there is no federal EITC, no qualifying child, and no property tax or rent for the real property tax credit, so the refundable total is $0." -us,scenario_005,federal_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,,"PolicyEngine applies the 2026 rules under OBBBA, which made the TCJA rules permanent. AGI is $536,764.50: wages less traditional 401(k) deferrals, where each spouse's $27,240 traditional+Roth request is cut to the $24,500 combined limit, pro-rating traditional deferrals to $20,825 each ($41,650 total); plus the $2,164 IRA deduction, the $3,000 capital-loss deduction, and no taxable state refund. The household takes the $32,200 standard deduction plus the $1,070 non-itemizer cash charitable deduction, because the phased-down SALT cap (~$30.9k) and the charity gift, which the 0.5%-of-AGI floor wipes out, cannot beat it. Tax is $89,095.92 of ordinary bracket tax, plus 15% on $77,920 of qualified dividends ($11,688), plus 3.8% NIIT on $150,578.34. The wrong answers did one or more of these: applied TCJA-sunset law (pre-TCJA brackets, Pease, personal exemptions, AMT), invented mortgage interest from a balance with no interest given, left out NIIT, subtracted ESI premiums from wages, or, among the near-misses, missed the 401(k) pro-rating, the refund exclusion, and the non-itemizer charitable deduction. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." +us,scenario_005,federal_income_tax_before_refundable_credits,48,reference_engine_defect,taxable_income_or_deductions,False,"The reference's $5,163.84 above-the-line deduction is the $3,000 capital-loss deduction plus the full $2,164 of traditional IRA contributions. Under IRC §219(g), the IRA deduction phases out completely for married joint filers above about $149,000 MAGI in 2026 when both spouses actively participate in an employer plan. Both spouses here make 401(k) deferrals and have about $537k of MAGI, so their IRA deduction should be $0, which points to PolicyEngine not applying the §219(g) active-participant phase-out.","The reference reaches AGI of $536,764 from four steps. First, it allows only $41,650 of traditional 401(k) deferrals: the $24,500 elective-deferral limit is split pro rata across each spouse's $27,240 of combined Roth and traditional deferrals. Second, it counts no state-refund income. Third, it takes the $3,000 capital-loss deduction. Fourth, it deducts the $2,164 of IRA contributions. It then subtracts the $32,200 standard deduction plus the $1,070 non-itemizer cash charitable deduction; no mortgage interest is listed, so none is deductible. That leaves $503,495 of taxable income, taxed as $89,096 ordinary tax, plus $11,688 (15% on $77,920 of qualified dividends), plus $5,722 of net investment income tax (NIIT) on $150,578, with no AMT. The closest models differed from the frozen reference in the non-itemizer charitable deduction, 401(k) allocation, refund treatment, or IRA deduction. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g); the frozen reference improperly allows it. The record leaves refund taxation and excess 401(k) allocation unresolved. The far misses assumed the TCJA expired (Pease, AMT, uncapped SALT), invented mortgage interest, subtracted employer premiums from wages, or omitted NIIT included in the frozen reference; whether this output includes NIIT also remains unresolved. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_005,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_005,payroll_tax,43,llm_error,state_local_rule,False,,"The federal parts were easy to get right: Social Security is 6.2% of $180,000 plus 6.2% of the $184,500 2026 cap on the spouse's $250,000, totaling $22,599. Medicare is 1.45% of $430,000, or $6,235. Additional Medicare Tax is 0.9% of the $180,000 above the $250,000 MFJ threshold, or $1,620. The trap that separated the answers is California SDI. In 2026 it is 1.3% of all gross wages with no wage ceiling, so it is $5,590 on $430,000. Most models left SDI out, used the old 1.1%/1.2% rates, applied the wage cap that was repealed in 2024, or subtracted employer-sponsored insurance (ESI) premiums from the payroll wage base, which the engine taxes in full." +us,scenario_005,payroll_tax,44,llm_error,state_local_rule,False,,"The federal part is simple. Social Security is 6.2% of $180,000 for the head plus 6.2% of $184,500 (the 2026 wage base) for the spouse, giving $22,599. Medicare is 1.45% of $430,000, or $6,235, and the Additional Medicare Tax is 0.9% of the $180,000 above the $250,000 joint threshold, or $1,620. That makes $30,454 (the derivation's ""$28,454"" is a typo; its own components and the $36,044 total add up to $30,454). The step that separates right from wrong answers is California SDI: in 2026 it is 1.3% of all wages with no cap (the cap ended in 2024), so it adds $5,590 on the full $430,000 gross, with no reduction for employer health premiums. Most models left SDI out, used the 1.1% or 1.2% rate, applied an old SDI wage ceiling, used an older Social Security wage base, or cut payroll wages by the health premiums." us,scenario_005,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_005,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The answer depends on getting California taxable income right. Each spouse wants $23,154 traditional plus $4,086 Roth 401(k), or $27,240 in total, which is over the $24,500 2026 elective-deferral limit. PolicyEngine pro-rates the cap, so only about $20,825 of traditional deferral per spouse comes out of wages. That puts federal AGI at $543,076, and California AGI is $538,928 after the $4,148 SALT-refund subtraction. California itemized deductions are property tax $17,280, charity $2,574 and employee business expenses above the 2% floor ($2,986). There is no mortgage interest because none is listed, and the tax-exempt pension and interest are not added back. The 6% high-income limitation cuts these deductions to $20,520, so taxable income is $518,408 and tax is $41,089. Only $114 of the exemption credit survives the phase-out. Most models deducted the full $46,308 of 401(k) deferrals, made up mortgage interest, subtracted employer-paid premiums, skipped or misapplied the itemized limitation, or treated the exemption credit as either fully phased out or not phased out at all." +us,scenario_005,state_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"The engine's federal AGI of $543,076 excludes only $41,650 of traditional 401(k) deferrals. Each spouse asked for $23,154 traditional plus $4,086 Roth, which is $27,240 and over the 2026 $24,500 elective-deferral limit, so traditional deferrals scale down to $20,825 each. Subtracting the SALT refund gives CA AGI of $538,928. Itemized deductions cover property tax, charity, and employee expenses above the 2% floor; no mortgage interest is given, so none is deducted. The 6% high-income limitation cuts the total to $20,520, which leaves $518,408 of taxable income, $41,089 of tax, and a $114 exemption credit after phase-down. Most models instead used CA AGI near $534,270 by subtracting the full $46,308 of deferrals, imputed mortgage interest from the loan balance, skipped or over-applied the itemized limitation, or claimed exemption credits without the phase-down." us,scenario_005,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_007,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The correct answer needs three things. First, taxable SSDI is $13,784.12: provisional income is $35,280 + $9,642.50 = $44,922.50, and the taxable amount is the lesser of 85% × $19,285 and 0.85 × ($44,922.50 − $34,000) + $4,500. The tax-exempt pension is not counted in provisional income. Second, the 2026 single standard deduction is $16,100, because OBBBA made the TCJA rules permanent, so there is no personal exemption. Third, the 2026 brackets are 10% up to $12,400 and 12% above that, which gives $1,240 + 12% × $20,564.12 = $3,707.70. The wrong answers fall into four groups: some applied the SS taxation formula incorrectly (taking the 85% cap instead of the lesser amount, using only the 50% tier, adding the $284, or treating SSDI as untaxed); some used stale 2024/2025 deductions and brackets; some assumed the TCJA had expired, with personal exemptions and a 15% bracket; and some submitted a value that differs from their own calculation. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors." +us,scenario_007,federal_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"Provisional income is $35,280 + 50% × $19,285 = $44,922.50. The $284 tax-exempt pension is not tax-exempt interest, so it is not included. Taxable Social Security is the lesser of 85% of benefits ($16,392.25) and 85% × ($44,922.50 − $34,000) + $4,500, which equals $13,784.12. That makes AGI $49,064.12. Under OBBBA, the 2026 single standard deduction is $16,100 and the 10% bracket ends at $12,400, so tax on $32,964.12 of taxable income is $1,240 + 12% × $20,564.12 = $3,707.70, with no nonrefundable credits. The models went wrong in four main ways: they skipped the lesser-of step and taxed the full 85%, left SSDI out entirely, used 2024/2025 or pre-TCJA deductions and brackets, or submitted a number that didn't match their own working. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors." us,scenario_007,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_007,head_medicare_eligible,22,prompt_ambiguity,age_disability,False,,"The head is 56, so Medicare's age-65 test fails, and the only other route is disability entitlement. Under 42 U.S.C. 426(b), that route requires 24 months of SSDI entitlement, not just SSDI income in the current year. The household lists no disability status, no months of SSDI receipt, no ESRD and no ALS. The prompt says unlisted status inputs are false and unlisted numbers are 0, and a constant status across one tax year shows at most 12 months. All 22 models treated the $19,285 of SSDI income as proof of Medicare eligibility, either by assuming the 24-month wait was already met or by claiming PolicyEngine grants eligibility whenever SSDI is positive, so they answered Yes instead of No. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_007,head_medicare_eligible,23,prompt_ambiguity,age_disability,False,,"The head is 56, so the age-65 pathway is closed. Disability-based Medicare requires 24 months of SSDI entitlement (or ESRD/ALS). PolicyEngine tracks this with a months-receiving-SSDI input that is unlisted here, so it defaults to 0, and no disability or ESRD flag is set. Every model treated a positive Social Security disability income amount as proof of Medicare entitlement, and several assumed the 24-month waiting period was met. The prompt only says facts are constant within the tax year, which does not establish 24 prior months of receipt, so the head is not Medicare eligible. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_007,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax applies to wages and other FICA-covered earned compensation, not to Social Security disability benefits or private pension income. The prompt specifies no wage or self-employment income and requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and payroll tax is $0." us,scenario_007,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_007,state_income_tax_before_refundable_credits,47,reference_engine_defect,state_local_rule,False,,"Idaho starts from federal AGI of $49,064.12 and subtracts the $13,784.12 of federally taxable Social Security, leaving $35,280. The $16,100 standard deduction brings Idaho taxable income to $19,180. For 2026 Idaho taxes income at 5.3%, but the first $4,920 for a single filer is taxed at zero, so the tax is 0.053 × ($19,180 − $4,920) = $755.78. Most models got to $19,180 and then left out the zero-rate amount, used the old 5.695% or 5.8% rate, or used an out-of-date zero-rate amount. Others never subtracted Social Security, used the wrong standard deduction, or wrongly applied Idaho's retirement-benefits deduction to a private pension for someone aged 56. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)." +us,scenario_007,state_income_tax_before_refundable_credits,48,llm_error,state_local_rule,False,,"The reference takes federal AGI of $49,064.12 and removes the $13,784.12 of taxable Social Security. It also takes off a $2,080 subtraction for health insurance premiums the household paid that were not otherwise deducted, which gives Idaho AGI of $33,200. After the $16,100 standard deduction, Idaho taxable income is $17,100. Tax is 5.3% on the amount above the $4,811 zero-rate band, or $651.32, and the $10 permanent building fund tax brings the total to $661.32. Most models that got the Social Security subtraction and the $16,100 deduction right still landed on $1,016.54 or $761.56. They missed the health-premium subtraction, and many also missed the zero-rate band and the $10 building fund tax. The other models used the old 5.695%/5.8% rates, the expired $8,550 deduction, or a non-existent personal exemption, or they applied the retirement-benefits deduction to a private pension." us,scenario_007,state_refundable_credits,37,llm_error,thresholds_rates,False,,"The whole $155 is the Idaho Grocery Credit. It is fully refundable and has no income phase-out. It is reduced only for months when the person gets SNAP or is incarcerated, and this filer qualifies for all 12 months. Idaho's 2025 legislation (HB 231) raised the per-person amount for filers under 65 from $120 to $155 starting in tax year 2025. The wrong answers split into two groups. One group used outdated per-person amounts: $120 under 2023–2024 law, $100 under earlier law, or $140, which was the old age-65+ amount. The other group answered $0, treating the credit as nonrefundable, phased out by income, repealed, or nonexistent." us,scenario_008,child1_chip_eligible,9,llm_error,health_coverage,False,,"CHIP is a residual program: it covers children who do NOT qualify for Medicaid, so its income band has a floor at the state's children's Medicaid limit, not just a ~350% FPL ceiling. With household income of about $30,915 for a family of eight — roughly 54% of FPL — the 17-year-old clears New Jersey's Medicaid test in the OLDER_CHILD category, and that Medicaid eligibility forecloses CHIP. All nine models tested only the upper CHIP ceiling and read very low income as evidence for CHIP, when low income is exactly what routes the child to Medicaid instead." us,scenario_008,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A 17-year-old tax-unit dependent falls within New Jersey's OLDER_CHILD Medicaid category. The household's MAGI is 0.55 times FPL, below the category's income threshold, so the child is eligible without any additional eligibility trigger." @@ -60,21 +60,21 @@ us,scenario_008,child6_early_head_start_eligible,1,llm_error,categorical_eligibi us,scenario_008,child6_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Head Start requires both preschool-age categorical eligibility and satisfaction of the program's income or another qualifying eligibility pathway. Child 6 is age 1 and therefore belongs in the Early Head Start age category, not the preschool-age Head Start category; the household also exceeds the applicable income cutoff." us,scenario_008,child6_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New Jersey Medicaid applies a YOUNG_CHILD eligibility pathway to the 1-year-old dependent. At a MAGI income level of 0.55 times FPL, the child satisfies that category's income test and is eligible without any additional benchmark-triggering fact." us,scenario_008,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_008,federal_refundable_credits,42,llm_error,thresholds_rates,False,,"The reference has two parts. The first is the full 2026 three-or-more-child EITC maximum of $8,231, with no phase-out because AGI (about $30,632) is below the 2026 MFJ phase-out threshold. The second is a refundable CTC of 15% × (earned income − $2,500). Earned income here is $26,800 wages plus $4,000 SE income minus the $282.59 half-SE-tax deduction, or $30,517.41, so the refundable CTC is $4,202.61. Only five children count for the CTC because the 17-year-old is excluded. Five times the $1,700 refundable cap is not binding, tax liability is zero, and the AOTC is $0 because the household has no qualified expenses. Most models missed on at least one of these steps: they used the wrong EITC maximum or applied a phase-out that does not apply, used the expired pre-TCJA $3,000 ACTC threshold, used gross $30,800 or 0.9235×SE earned income, or added an AOTC with no expenses." +us,scenario_008,federal_refundable_credits,43,llm_error,credit_phaseout,False,,"The reference has two parts. The first is the full 2026 3+-child EITC maximum of $8,231, with no phase-out, because AGI of about $30,632 is below the 2026 joint-filer phase-out start. The second is refundable ACTC of 15% × (earned income − $2,500). Here, earned income is $26,800 in wages plus $4,000 of self-employment income minus the half-SE-tax deduction of $282.59, which gives $30,517.41 and an ACTC of $4,202.61. Only five children are under 17, but the $1,700-per-child cap ($8,500) does not bind, and zero tax liability does not reduce the refundable amount. Most near-misses made one of these errors: they used gross earned income of $30,800 or 0.9235 × SE income ($30,494); they used the pre-TCJA $3,000 ACTC threshold; they used the wrong EITC maximum (for example, the 2025 $8,046); they applied an EITC phase-out that does not apply; or they added a refundable AOTC even though qualified education expenses are zero." us,scenario_008,free_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Free school meals do not require an explicitly listed direct-certification indicator in the household inputs. The household qualifies through both routes established by the computation: its school-meal income is 55% of the federal poverty guideline, below the 130% free-tier threshold, and all household members are SNAP-eligible, which supplies categorical eligibility." us,scenario_008,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"New Jersey expanded Medicaid under the ACA. Any non-elderly adult with household MAGI at or below 138% FPL qualifies in the ADULT category, with no disability requirement. This household has 8 members. Its MAGI is about $30.8k: $26,800 in wages, plus $4,000 in self-employment income minus half of SE tax, plus $115 in interest. That works out to 0.55× FPL, well under the 138% limit, so the 42-year-old head is eligible. Every model answered No for one of three reasons: it assumed non-disabled adults are categorically excluded, it misjudged where the income falls against the threshold, or it made up an exclusion rule." us,scenario_008,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not conferred on every household member by the presence of an eligible infant or young child. The 42-year-old Head is not pregnant, postpartum, or breastfeeding and is not an infant or child under age 5, so satisfying the household income limit does not make the Head eligible." us,scenario_008,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_008,payroll_tax,45,llm_error,state_local_rule,False,,"Federal employee FICA on the spouse's $26,800 of wages is $2,050.20. Almost every model got that part right. The models differ on New Jersey's four mandatory employee contributions, all charged on the full $26,800 at 2026 rates: TDI 0.19% ($50.92), FLI 0.23% ($61.64), UI 0.3825% ($102.51) and workforce/SWF 0.0425% ($11.39), which add $226.46. Most models left out the NJ contributions entirely. Others used the 2024 rates (TDI 0%, FLI 0.09%) or the 2025 rates (TDI 0.23%, FLI 0.33%), or dropped one of the four components." +us,scenario_008,payroll_tax,46,llm_error,state_local_rule,False,,"Every model computed the federal employee FICA correctly: 7.65% of the spouse's $26,800 wages is $2,050.20. The answers split on New Jersey's mandatory employee payroll contributions, which add $226.46. That is 0.845% of wages, made up of UI at 0.3825% ($102.51), the workforce fund at 0.0425% ($11.39), TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64). About half the models left out the NJ contributions entirely. The rest left out some components or used the wrong 2026 TDI/FLI rates, most often 0% TDI with 0.09% FLI, or the older 0.23% TDI with 0.33% FLI." us,scenario_008,reduced_price_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a generic indicator that income is below the 185% FPL ceiling. This household is at 55% of the federal poverty guideline and also meets categorical eligibility, so it receives the superior free-meals tier and reduced_price_school_meals_eligible equals 0." us,scenario_008,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax is 15.3% of 92.35% of the $4,000 net self-employment income: $4,000 × 0.9235 × 0.153 = $565.18. The deduction for one-half of self-employment tax is an income-tax deduction and does not reduce the self-employment tax liability itself." -us,scenario_008,snap,46,llm_error,thresholds_rates,False,,"New Jersey's broad-based categorical eligibility makes the household eligible, so the spouse's $69,500 in bank assets does not count. The benefit then follows from FY2026 parameters. Gross income of $2,576.25 minus the $299 standard deduction (6+ people) and the $513.33 earned income deduction gives $1,763.92, which rounds to $1,764. Thirty percent of that is $529.20, rounded up to $530, and the $1,789 maximum for eight people minus $530 gives $1,259 a month, or $15,108 a year. There is no shelter, utility, or medical deduction. Most wrong answers either used outdated or invented maximum allotments and standard deductions, or skipped the rounding of net income and the 30% contribution and landed at about $1,260 a month. The $0 answers wrongly applied an asset test or income test." +us,scenario_008,snap,47,llm_error,thresholds_rates,False,,"The household is categorically eligible through New Jersey's TANF non-cash BBCE, so the $69,500 in bank assets does not count. Monthly gross income is $2,576.25. Subtracting the $299 standard deduction for 6+ people and the $513.33 earned income deduction gives net income of $1,763.92, which rounds to $1,764. There is no shelter deduction: rent is $579/month, below half of adjusted income, and no utilities are listed. There is no medical deduction either, because the spouse's over-the-counter health costs are $25/month, under the $35 threshold. The 30% contribution of $529.20 rounds up to $530, and $1,789 (the FY2026 maximum for 8 people) minus $530 is $1,259 a month, or $15,108 a year. Models missed this by applying an asset test, using stale or wrong maximum allotments and standard deductions, adding shelter deductions that don't apply, subtracting a monthly deduction from annual income, or skipping the monthly rounding of net income and the contribution." us,scenario_008,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"New Jersey adopted the ACA adult expansion, so a non-elderly adult 19-64 who is not a tax-unit dependent qualifies on the MAGI pathway at or below 138% FPL, with no categorical hook, no asset test, and no benefit-receipt disqualifier. The test compares household MAGI to the FPL for the full household size: roughly $30,900 of wages, self-employment income and interest against the eight-person FPL is 0.55 x FPL, less than half the 1.38 limit. Housing assistance is excluded from MAGI and the $69,500 in bank accounts is irrelevant because MAGI categories carry no resource test. Each wrong answer either skipped this income test entirely or overrode it with a disqualifier that does not exist." us,scenario_008,spouse_medicare_eligible,6,llm_error,age_disability,False,,"Medicare eligibility here turns on a single age test: the person must be 65 or older, and the Spouse is 41. The trap is the `is_disabled` flag: every wrong model converted it into a disability-based Medicare pathway, but real-world under-65 Medicare entitlement runs through 24 months of SSDI cash benefits (or ESRD/ALS), none of which is present — the household reports no Social Security disability income, and the prompt directs that unlisted income and status inputs be treated as 0/false. A disability flag alone establishes neither SSDI entitlement nor the 24-month qualifying period, so it cannot produce Medicare eligibility for a 41-year-old." us,scenario_008,spouse_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific: an adult must be pregnant, breastfeeding, or within the applicable postpartum period, while infants and children qualify only through their own age category. Low household income and the presence of a young child do not make the child's mother or caretaker independently eligible." us,scenario_008,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_008,state_income_tax_before_refundable_credits,41,llm_error,state_local_rule,False,,"New Jersey taxable income here is NJ gross income ($30,915 = wages + full $4,000 SE profit + $115 interest, with no half-SE-tax adjustment) minus exemptions only. The exemptions are $1,000 for each spouse, $1,500 for each of the 6 dependents, and a single $1,000 blind-or-disabled exemption for the spouse, who gets only one even though they are both blind and disabled. That leaves $18,915 × 1.4% = $264.81. Models mostly went wrong in four ways: they claimed separate blind and disabled exemptions, used the wrong exemption amounts, subtracted the $50 property-tax credit (it is refundable, so it doesn't belong in this output) or the 18%-of-rent deduction (not taken, because the household takes the credit), or treated OTC health expenses as a deductible medical expense. Others zeroed the tax using a standard deduction or filing-threshold rule that New Jersey doesn't have." -us,scenario_008,state_refundable_credits,47,llm_error,state_local_rule,False,,"The reference adds three New Jersey credits. The first is a $3,292.40 NJ EITC, equal to 40% of the full $8,231 federal maximum for 3+ children; at about $30.5k of earned income, the couple has not reached the MFJ phase-out. The second is a $2,500 NJ child tax credit for the two children age 5 and under, using the schedule enacted June 30, 2026 and keyed to NJ taxable income (well under $30,000 after exemptions), not gross income. The third is a $50 property tax credit for the renter household. Correctly applying the older $1,000-per-child CTC schedule gives only $5,342.40, so even flawless pre-enactment knowledge misses by $500. Most models also made their own errors on top of this: they phased out the federal EITC too early, chose the CTC tier from gross income, used outdated EITC maximums, left out the $50 credit, or denied that NJ has refundable credits at all. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned reference_later_law; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors." +us,scenario_008,state_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"The reference is a straight NJ calculation. NJ gross income is $30,915 (wages, Schedule C profit and interest, with no federal half-SE-tax adjustment). Exemptions total exactly $12,000: $1,000 for each spouse, $1,500 for each of the 6 dependents, and a single $1,000 blind-or-disabled exemption for the spouse, who is both blind and disabled but gets only one. Taxable income is $18,915, all taxed at 1.4%, for $264.81. Nothing else reduces it: NJ has no standard deduction, OTC health spending does not count toward the NJ medical deduction, no renter property-tax deduction applies to this housing-assisted household, and the $50 property-tax credit and the NJ child credit/EITC are refundable, so they are excluded. Most models either miscounted the exemptions or subtracted one of these items that does not apply." +us,scenario_008,state_refundable_credits,48,llm_error,state_local_rule,False,,"The reference has three parts. The first is the NJ EITC: 40% of the full $8,231 federal EITC for three or more children, which is $3,292.40. The second is a $2,500 NJ Child Tax Credit for the two children age 5 and under, using the higher low-income schedule New Jersey enacted on June 30, 2026. The third is the $50 property tax credit, which applies because the household pays $6,949 in rent. The best answers got the EITC right but still used the old $1,000-per-child schedule. Many others tested the CTC against gross income (about $30,915) instead of NJ taxable income, which falls well under $30,000 after the $1,000 personal and $1,500-per-dependent exemptions, so they landed in the $800 or $400 bracket. Others left out the CTC, the $50 credit, or every refundable credit. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned reference_later_law; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors." us,scenario_008,tanf,1,llm_error,thresholds_rates,False,,"New Jersey's WorkFirst NJ TANF is not a residual top-up: it is gated on countable income falling below the family-size eligibility standard tied to the maximum grant (roughly $770/month for a household of eight) and on countable resources at or below the $2,000 limit. This household's $26,800 in wages plus $4,000 of self-employment income is about $2,576/month, which clears the standard by a wide margin even after New Jersey's 50% earned-income disregard, and the $69,500 bank balance independently fails the resource test. Six children plus a blind and disabled spouse make the household read as categorically needy, which is the trap; family size raises the grant schedule but nowhere near enough to admit income at this level. The grant is therefore $0." us,scenario_009,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"AGI is $105,900. That is $60,000 of gross wages, plus $20,400 of taxable pension, plus $25,500 of taxable Social Security, which is the 85% cap because provisional income of $95,400 is far above $44,000. The answer depends on using the OBBBA-permanent TCJA parameters for 2026: a $32,200 MFJ standard deduction, a 10% bracket up to $24,800 and 12% above that. Taxable income of $73,700 then gives $2,480 + $5,868 = $8,348. Most models went wrong in one of three ways: they assumed TCJA expired (personal exemptions and a 15% bracket), they used inflation-projected pre-OBBBA figures ($30,000–$30,900 standard deduction, $23,850–$24,500 bracket), or they subtracted the ESI premium from wages, which are already given as gross W-2 income. Several models worked out $8,348 or close to it and then submitted an unrelated number." us,scenario_009,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -92,14 +92,14 @@ us,scenario_012,child1_head_start_eligible,1,parse_contract_failure,other,False, us,scenario_012,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi evaluates a 10-year-old dependent through the Medicaid older-child MAGI category. Household MAGI is 0.81 times FPL, below the applicable eligibility threshold, so the category and income tests produce eligibility." us,scenario_012,child1_wic_eligible,4,llm_error,age_disability,False,,"To get WIC, a person must fit a category: pregnant, postpartum or breastfeeding woman, infant, or child under 5. In federal rules, the child category ends at the 5th birthday. Child 1 is 10, so the child fails the categorical test no matter how low the household income is, which makes child1_wic_eligible No. Every wrong model let low income decide the answer, or used the wrong age cap, or wrote that the child failed but still submitted 1." us,scenario_012,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_012,federal_refundable_credits,37,llm_error,thresholds_rates,False,,"This is a married couple filing jointly with one qualifying child (age 10) and $22,000 of earnings. That income is above the end of the one-child EITC phase-in and below the joint phase-out start, so the household gets the full 2026 one-child maximum of $4,427. Federal income tax is $0. Under the OBBBA-permanent CTC rules, the refundable ACTC is min($1,700 refundable cap, 15% × ($22,000 − $2,500) = $2,925) = $1,700, for a total of $6,127. Most wrong answers either used the 2025 EITC maximum ($4,328) or another un-indexed figure, or got the ACTC wrong in one of three ways: the TCJA-sunset $1,000 credit, the full $2,000/$2,200 credit treated as refundable, or $1,800. A few also miscounted children by including the pregnancy, or applied phase-outs that do not reach $22,000." +us,scenario_012,federal_refundable_credits,38,llm_error,thresholds_rates,False,,"The household files jointly with exactly one qualifying child; the unborn child and the 18-year-old spouse do not count. Its $22,000 of earnings is above the one-child EITC phase-in end (about $13,020) and below the 2026 MFJ phase-out start, so it gets the full 2026 maximum EITC of $4,427 (34% × $13,020). Tax liability is zero, so the ACTC is 15% × ($22,000 − $2,500) = $2,925. That is capped at the 2026 refundable limit of $1,700, which OBBBA kept even though it raised the CTC to $2,200. Most models missed by using 2025 EITC figures ($4,328) or invented inflation adjustments, by applying the $1,000 post-sunset ACTC instead of the $1,700 limit, by using a $1,800 cap, or by treating the full CTC as refundable." us,scenario_012,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"This three-person Mississippi household has $22,000 in wages, which is 81% of the federal poverty guideline. That is far below the 130% free-meal cutoff. The household also receives about $407.90 a month in SNAP, which makes the 10-year-old categorically eligible for free meals, and PolicyEngine treats a 10-year-old as enrolled in K-12. So the engine puts the household in the FREE tier. The wrong answers came from three different errors: one model never ran the income or SNAP test, one treated the child's school enrollment as an unlisted input set to false, and one wrongly claimed income was above 130% of the poverty guideline." us,scenario_012,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Mississippi Medicaid eligibility for a nondisabled 40-year-old adult requires qualification through a defined categorical pathway; income below 138% FPL does not independently establish eligibility. The head’s MAGI is 0.81 FPL, but the Medicaid category is NONE because the head satisfies none of Mississippi’s available pathways." us,scenario_012,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not extended to every member of a household containing a pregnant participant. The pregnant spouse satisfies the categorical requirement, but the 40-year-old head is neither pregnant, postpartum, breastfeeding, an infant, nor a child under age five, so the head is not WIC-eligible despite household income below the limit." us,scenario_012,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_012,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_012,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. At a school-meal FPG ratio of 0.81 and with categorical eligibility, the household is classified as FREE, so it receives no reduced-price support." -us,scenario_012,snap,46,llm_error,taxable_income_or_deductions,False,,"The SNAP unit has three people. Pregnancy does not add a member. Monthly net income is $1,833.33 − $209 standard deduction − $366.67 earned-income deduction = $1,258. Shelter cost is only the $450 rent. The $411 energy subsidy does not give this household, which has no elderly or disabled member, a standard utility allowance under the 2026 rules, so there is no excess shelter deduction because $450 is less than half of net income ($629). The benefit is $785 minus 30% of net income rounded up to $378, which is $407 a month or $4,884 a year. Most wrong answers either added a utility allowance triggered by the energy subsidy, counted the unborn child as a fourth member, used older FY2025 deduction and allotment amounts ($204/$768), or skipped SNAP's rounding rules ($407.70 instead of $407)." +us,scenario_012,snap,47,llm_error,taxable_income_or_deductions,False,,"The SNAP unit has three members; the unborn child is not counted. Monthly gross income is $1,833.33. Subtracting the $209 standard deduction and the $366.67 earned income deduction leaves $1,258 after rounding. There is no excess shelter deduction because the $411 energy subsidy does not give a household with no elderly or disabled member the standard utility allowance under the 2026 rules, and no utility expenses are listed. Rent alone ($450) is below half of net income ($629). The 30% contribution of 377.4 rounds up to $378, so the allotment is $785 − $378 = $407 a month, or $4,884 a year. Most wrong answers either added an SUA-driven shelter deduction because of the energy subsidy, counted the pregnancy as a fourth member, used outdated FY2025 or earlier standard deductions and maximum allotments, or skipped SNAP's whole-dollar rounding and landed at about $4,890 to $4,896." us,scenario_012,spouse_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person not already qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under Mississippi's OLDER_CHILD category, so that Medicaid eligibility precludes CHIP regardless of pregnancy status or how far household income falls below a CHIP-related threshold." us,scenario_012,spouse_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi Medicaid applies an OLDER_CHILD eligibility pathway to the 18-year-old spouse. The spouse's MAGI of 0.67 times FPL is below the applicable income threshold, so this category produces eligibility despite the spouse not being a tax-unit dependent." us,scenario_012,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC covers pregnant individuals who satisfy the program's income test. The spouse is pregnant, and the household's approximately $22,000 annual income is below the applicable 185% federal-poverty income limit, so the spouse meets both categorical and financial eligibility requirements." @@ -107,20 +107,20 @@ us,scenario_012,ssi,1,parse_contract_failure,missing_output,False,,All wrong res us,scenario_012,state_income_tax_before_refundable_credits,9,llm_error,thresholds_rates,False,,"Mississippi taxable income for this married-joint household is $22,000 minus the $4,600 MFJ standard deduction, the $12,000 MFJ personal exemption, and the $1,500 exemption for the one dependent child, which leaves $3,900. Mississippi taxes the first $10,000 of taxable income at 0% and applies its flat rate (4.0% in 2026) only above $10,000, so the joint liability is $0. PolicyEngine's per-spouse allocation gives the head $118 (4% of $12,950 minus $10,000), but the unit-level result is $0. Four models computed the $3,900 base correctly and then taxed it from the first dollar. The rest also got the deductions or exemptions wrong, and one of them used another state's credit structure." us,scenario_013,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The trap is the Social Security provisional-income test: half of the $23,736 benefit ($11,868) plus the $6,720 pension, $16 of dividends and $72 of capital gain gives provisional income of $18,676, below the $25,000 first base amount for a single filer, so none of the Social Security is includible. AGI is therefore $6,808, which is under the single standard deduction before any age-65 additional amount or the OBBBA senior deduction is even reached, leaving taxable income and tax of $0. A second, independent guardrail is that federal income tax before refundable credits is floored at zero — nonrefundable credits can only zero out liability, never drive it negative." us,scenario_013,head_medicaid_eligible,3,llm_error,age_disability,False,,"The head is 80, so the 138% FPL MAGI adult expansion group (ages 19-64) is closed to them. The only route left is the aged/blind/disabled pathway, which counts Social Security in full. Social Security of $23,736 plus the $6,720 pension and $88 of investment income comes to about $30,544, or 1.91x FPL. That is far above the SSI benefit rate and the 100% FPL aged/disabled standard, and with SSI at 0 there is no SSI-linked eligibility either, so the engine returns category NONE. All three models answered Yes. One asserted eligibility without testing income, one excluded most of the Social Security, and one used AGI, which leaves out the nontaxable Social Security." -us,scenario_013,snap,47,llm_error,categorical_eligibility,False,,"Starting in March 2026, Arizona's broad-based categorical eligibility (BBCE) covers households with gross income up to 200% FPL. That limit is $2,608.33/month, rising to $2,660 from October. This household's gross income is $2,539.33/month (Social Security, pension and dividends), which is under that limit. BBCE treats both the asset test and the net income test as met, so the $58,700 in the bank and the $2,138 net income no longer disqualify it. The benefit formula ($298 minus 30% of net income) comes out below zero, but a categorically eligible one-person household still gets the $24 monthly minimum. That gives $24 × 10 months (March–December) = $240. Every model answered $0. They either applied the regular asset and net income tests, used Arizona's old 185% BBCE limit, or treated a negative benefit formula as ineligibility instead of applying the minimum benefit." +us,scenario_013,snap,48,llm_error,categorical_eligibility,False,,"This household fails SNAP's ordinary tests: net income is $2,138 a month and it has $58,700 in the bank. But from March 2026, Arizona's broad-based categorical eligibility (BBCE) covers households with gross income up to 200% of the poverty line, which is $2,608.33 a month ($2,660 from October). The household's gross income of $2,539.33 is under that limit, so the income and asset tests count as met. Because a categorically eligible one-person household gets the $24 minimum allotment even when $298 minus 30% of net income comes out negative, the benefit is $24 for each month from March through December, or $240 a year. Every model either applied the ordinary net-income, gross-income or asset test, used Arizona's old 185% BBCE limit, or stopped at the zero formula result without applying the minimum allotment." us,scenario_013,state_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Arizona’s standard deduction eliminates the remaining taxable income after excluding Social Security, so applying the 2.5% rate directly to gross dividends and capital gains is incorrect. The resulting Arizona income tax before refundable credits is zero." -us,scenario_013,state_refundable_credits,37,llm_error,state_local_rule,False,,"Arizona's Credit for Increased Excise Taxes (A.R.S. §43-1072.01) is refundable and pays $25 for the taxpayer plus $25 for each dependent, capped at $100. The single-filer limit is $12,500 of income. This person's federal AGI is $6,736 because none of the Social Security is taxable at their income level, so one $25 credit applies. They don't need earned income, dependents, tax liability, rent, or property tax to get it. Nearly every model missed this credit, ruled it out with made-up requirements, or misread the income test. One model counted age and disability exemptions as extra credit units." +us,scenario_013,state_refundable_credits,38,llm_error,state_local_rule,False,,"The $25 comes from Arizona's Credit for Increased Excise Taxes (A.R.S. §43-1072.01). It is fully refundable and pays $25 for each person in the tax unit (taxpayer, spouse, dependents), up to $100. A single filer qualifies with Arizona AGI at or below $12,500. Social Security is excluded here: provisional income of about $18,676 is under the $25,000 base, so none of it is federally taxable, and Arizona exempts it anyway. That leaves Arizona AGI of about $6,736, well under the limit, so this one-person unit gets $25. The credit does not depend on tax liability, earned income, dependents, rent, or property tax. Almost every model either said Arizona has no applicable refundable credit or ruled it out for those wrong reasons. One model counted age and disability exemptions, which this credit does not use." us,scenario_014,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The only taxable income is the spouse's $88,927.65 in wages, and AGI equals that full amount. Veterans benefits are excluded, and the listed ESI premium is not subtracted from the gross wage input. Under OBBBA's permanent 2026 parameters, the MFJ standard deduction of $32,200 leaves taxable income of $56,727.65. Tax is then 10% of the first $24,800 plus 12% of the remaining $31,927.65, which equals $6,311.32, and no nonrefundable credits apply because there are no dependents. Most models went wrong in one of four ways: they assumed the TCJA had sunset (bringing back personal exemptions and a 15% bracket), used stale 2025 deduction and bracket figures, subtracted the ESI premiums from wages, or did the math correctly and then submitted a different number." us,scenario_014,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"Federal refundable credits are zero because this childless joint tax unit has income far above the EITC phase-out range and has no qualifying child for refundable CTC. No education, rebate, or refundable payroll-tax-credit facts create another refundable federal credit." us,scenario_014,head_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"Medicaid eligibility for the head uses household tax-unit MAGI rather than only the head's earned income. The household's MAGI is approximately $183,584, or 4.11 times FPL, exceeding every applicable West Virginia Medicaid income limit, and the head has no alternative categorical pathway." us,scenario_014,payroll_tax,16,llm_error,payroll_tax_base,False,,"Only the spouse has wages, so the correct answer is the 7.65% employee FICA rate applied to her full $88,928 gross wages: $5,513.51 Social Security plus $1,289.45 Medicare, or $6,802.96. The facts do not say she pays premiums through a salary-reduction (Section 125) plan, so the $14,717 in employer-sponsored insurance premiums does not come out of the FICA wage base. Veterans benefits are not wages, and West Virginia has no employee state payroll tax. The largest group of wrong answers (7 models) assumed a cafeteria plan that the facts never mention and taxed only $74,211. Most of the rest applied 7.65% to $88,928 and then invented an extra tax, made an arithmetic slip, or submitted a number different from their own total." -us,scenario_014,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"West Virginia AGI is the spouse's $88,928 of wages. The veterans benefits are excluded, and the employer-sponsored insurance premium input does not reduce wages. WV has no standard deduction, so the only subtraction is two $2,000 personal exemptions, which leaves $84,927.65 of taxable income. The deciding step is the 2026 MFJ rate schedule. It is the 2025 schedule (2.22/2.96/3.33/4.44/4.82%) cut by another 5%, to about 2.11/2.81/3.16/4.22/4.58%, which yields $3,092.19 rather than $3,255 (2025 rates) or $3,460 (2024 rates). Many models also changed the base: they subtracted the $14,717 ESI premium to get $74,211, made up a standard deduction, or applied invented credits." +us,scenario_014,state_income_tax_before_refundable_credits,48,llm_error,thresholds_rates,False,,"West Virginia taxable income is federal AGI of $88,927.65, which is the spouse's wages only. The veterans benefits are excluded and the $14,717 employer-sponsored insurance premium is not subtracted again. WV has no standard deduction, so the only subtraction is two $2,000 personal exemptions, leaving $84,927.65. The 2026 MFJ schedule cuts the 2025 rates by another 5%, to 2.11% / 2.81% / 3.16% / 4.22% / 4.58% at breakpoints of $10k / $25k / $40k / $60k: 211 + 421.50 + 474 + 844 + 0.0458 × 24,927.65 = $3,092.19. Most models got the $84,928 base right but used the 2025 schedule (2.22–4.82%, giving $3,255.03) or the 2024 schedule (2.36–5.12%, giving $3,459.81). Others shrank the base by subtracting ESI premiums or inventing standard deductions." us,scenario_015,federal_income_tax_before_refundable_credits,36,llm_error,thresholds_rates,False,,"The correct calculation is short. Start with wages of $45,000 minus the $1,042 traditional 401(k) deferral, add $2,000 of interest, subtract the $3,000 capped capital loss and the $49 IRA deduction, and AGI is $42,909. Subtract the $16,100 single standard deduction for 2026 (OBBBA made the TCJA structure permanent, so there is no personal exemption) to get taxable income of $26,809. Tax at 10% up to $12,400 and 12% above that is $1,240 + $1,729.13 = $2,969.13. No nonrefundable credits apply: there are no dependents, AGI is above the Saver's Credit limit, and the elderly/disabled credit is phased out. Most models got AGI right and then missed the answer in one of four ways: they used 2024 or 2025 standard deductions and bracket edges ($14,600, $15,000 or $15,750; $11,600, $11,925 or $12,150), assumed the TCJA had expired and applied a personal exemption with a 15% bracket, made up deductions such as mortgage interest or pre-tax health premiums, or made arithmetic and transcription errors after reaching about $2,969." us,scenario_015,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A single adult with no dependent children or qualifying education expenses receives neither the refundable Child Tax Credit nor the refundable American Opportunity Credit. At approximately $43,450 of income, the childless EITC is also zero, and all other evaluated refundable-credit components are zero, yielding total federal refundable credits of $0." us,scenario_015,head_chip_eligible,1,parse_contract_failure,other,False,,"CHIP eligibility runs through categorical gates first: coverage is for children under 19 and, at state option, pregnant women, so a 36-year-old head never reaches the income test — PolicyEngine assigns chip_category = 3 (categorically ineligible) and is_chip_eligible = False. The ~$43,450 household income and Indiana's Hoosier Healthwise/CHIP income thresholds are irrelevant to a person who fails the age/pregnancy gate, and the separate Medicaid pathways also return NONE for a non-aged, childless adult at that income in a state whose adult expansion he clears on neither disability-SSI nor MAGI grounds. The single wrong answer stated this categorical rule correctly in prose and then emitted the opposite binary." us,scenario_015,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish Medicaid eligibility; the person must satisfy a specific Indiana eligibility pathway and its applicable requirements. The head falls into Medicaid category NONE, receives no SSI, is not a dependent, and has MAGI at 2.69 times FPL, so no MAGI, disability-related, or state-specific pathway confers eligibility." us,scenario_015,head_medicare_eligible,4,llm_error,age_disability,False,,"A person under 65 becomes eligible for Medicare on disability grounds only after 24 months of entitlement to Social Security Disability Insurance, or through the ESRD or ALS pathways. Being disabled is not enough on its own. The head is 36, has no Social Security disability benefits, and has no ESRD or ALS status. That means they don't meet the age-65 test or any disability-entitlement pathway, so they are not Medicare eligible. All four models treated the bare 'is disabled' flag as automatic Medicare eligibility and skipped the SSDI-receipt requirement." -us,scenario_015,local_income_tax,47,prompt_ambiguity,state_local_rule,False,,"The prompt describes local_income_tax by listing only NYC, Philadelphia, Kansas City and St. Louis taxes. PolicyEngine's local_income_tax variable also includes Indiana county income tax. No county was given, so the engine used Adams County and applied its 1.6% rate to Indiana AGI of $41,909.38: federal AGI of $42,909.38 (after the $3,000 capital-loss deduction and the $48.69 IRA deduction) minus the $1,000 exemption, which gives $670.55. Every model answered $0, either because it read the four-city list as the complete scope or because it wrongly said Indiana has no local income tax. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))." +us,scenario_015,local_income_tax,48,prompt_ambiguity,state_local_rule,False,,"The prompt defines local_income_tax by listing only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax. PolicyEngine's local_income_tax variable also includes Indiana county income tax. No county was given, so the engine used its default, Adams County, at 1.6%: 1.6% × Indiana AGI of $41,909.38 (federal AGI $42,909.38 minus the $1,000 exemption) = $670.55. That leaves two readings. Reading (a): the output covers only the four listed city taxes, which gives $0. Reading (b): the output is PolicyEngine's full local_income_tax, which includes the Indiana county tax and gives $670.55. Every model chose reading (a), and a few also wrongly said Indiana has no local income tax at all. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))." us,scenario_015,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $45,000 of gross wages: 6.2% Social Security ($2,790, well under the 2026 wage base) plus 1.45% Medicare ($652.50) gives $3,442.50. Indiana has no mandatory employee state payroll tax, and wages are far below the $200,000 Additional Medicare Tax threshold. The most common wrong move was shrinking the FICA base. Four models treated the listed $2,080 health insurance premiums as a Section 125 pre-tax salary reduction, and one subtracted the traditional 401(k) deferral, even though elective 401(k) deferrals stay subject to FICA. The rest reached or approached $3,442.50 and then broke the total with arithmetic slips or made-up add-ons." us,scenario_015,state_income_tax_before_refundable_credits,33,llm_error,thresholds_rates,False,,"Indiana's 2026 tax is its flat 2.95% rate applied to federal AGI minus a single $1,000 personal exemption. Federal AGI is $42,909: wages of $45,000, minus the $1,042 traditional 401(k), minus the $49 IRA deduction, plus $2,000 interest, minus the $3,000 capped capital loss. Workers' compensation is excluded. That gives $41,909 × 2.95% = $1,236. Most models missed for one of three reasons. Some used the wrong 2026 rate (3.0%, 3.05%, 2.85%, 2.9%, 2.97%, 3.025%). Some invented exemptions or deductions that do not exist: a disability exemption for a non-blind taxpayer under 65, a health-premium deduction, a child-support deduction, or a standard deduction. Others built AGI incorrectly, for example by deducting the full $4,510 loss, including workers' comp, or skipping the 401(k) exclusion." us,scenario_015,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -140,15 +140,15 @@ us,scenario_016,ssi,1,parse_contract_failure,missing_output,False,,All wrong res us,scenario_016,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_016,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_016,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_018,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The calculation is simple. AGI is $61,592, the 2026 single standard deduction is $16,100, and taxable income of $45,492 is taxed at 10% up to $12,400 and 12% above that, giving $5,211.05. There are no nonrefundable credits. Most models missed because they used stale or invented parameters: the 2024/2025 standard deductions ($14,600, $15,000, $15,750), old 10% bracket ceilings ($11,925, $12,200, etc.), or a TCJA sunset with personal exemptions that OBBBA prevented. The rest added deductions that the facts don't support: a $6,760 overtime-premium deduction, a $1,625 car-loan interest deduction, double-counted medical premiums that pushed itemized deductions above the standard deduction, or Saver's/Child Tax Credits this single, childless filer with $61.6k AGI can't claim." +us,scenario_018,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference is a plain 2026 single-filer calculation under current (OBBBA) law. AGI is $61,592.09 after the traditional 401(k), the deductible IRA contribution and the $50 of interest. Subtracting the $16,100 standard deduction leaves taxable income of $45,492.09. Tax is 10% up to $12,400 plus 12% on the rest, or $5,211.05, and no nonrefundable credits apply. Models went wrong in five main ways: (1) they used 2024, 2025 or projected standard deductions and bracket breakpoints instead of $16,100 and $12,400; (2) they assumed TCJA had expired, bringing back personal exemptions and the 15%/25% brackets; (3) they claimed an overtime-premium deduction, even though wages of $68,005 are about $26 × 50 × 52 and include no time-and-a-half premium; (4) they claimed the car-loan interest deduction without a qualifying new vehicle; (5) they inflated itemized medical deductions or claimed credits the filer can't get." us,scenario_018,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_018,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $68,005 of wages here: $4,216.31 of Social Security tax plus $986.07 of Medicare tax equals $5,202.38. The listed health-insurance premiums were not identified as employee pre-tax salary-reduction contributions, and Arizona's lack of a separate state payroll tax does not remove federal employee FICA." us,scenario_018,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_018,state_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"Arizona taxes Arizona AGI, here the federal AGI of $61,592.09 (wages minus the traditional 401(k), plus interest, minus the IRA deduction), at a flat 2.5%. The Arizona standard deduction is $16,100 for a single filer in 2026: the $15,750 set by HB 4168, indexed the same way as the federal amount. That beats the $10,637.30 of itemized deductions and leaves taxable income of $45,492.09 and tax of $1,137.30. Most models got the AGI and the 2.5% rate right but used an old or guessed standard deduction ($13,790–$15,750). Others added an exemption a 53-year-old doesn't get, used repealed graduated brackets, took the wrong deduction path, or submitted a number that doesn't match their own work." us,scenario_018,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_020,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,other,False,,"The step that decides this case is whether to itemize. Texas has no state income tax, so the filer can deduct general sales tax from the IRS optional sales tax table, about $3,256. Added to the $13,834 of real estate taxes, that makes $17,090.34 of SALT. It sits well under the 2026 OBBBA cap of $40,400 and beats the $16,100 standard deduction. Once the filer itemizes, the $196 gift falls below the 0.5%-of-AGI floor and the non-itemizer charitable deduction is lost. The other required steps are: deduct the $5,709 of alimony above the line (AGI $300,150), treat the overtime deduction as fully phased out, disallow unreimbursed employee expenses (OBBBA made the suspension permanent, so there is no TCJA sunset), and add the $222.64 NIIT. That gives taxable income of $283,060 and tax of $68,056.71. The strongest models got everything except the sales tax deduction and landed $277–$347 too high. Many others assumed a pre-TCJA sunset, applied a $10,000 SALT cap, dropped the alimony deduction, or subtracted ESI premiums from wages. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)." +us,scenario_020,federal_income_tax_before_refundable_credits,48,prompt_ambiguity,other,False,,"Texas has no state income tax, so the SALT itemized deduction uses the general sales tax amount instead. The frozen reference projects the sales-tax amount from the IRS's 2023 table, making SALT $17,090.34 and causing it to itemize and disallow the $196 gift under the 0.5%-of-AGI floor. The release's sales tax convention (c_irs_sales_tax_2025) instead uses the IRS 2025 Texas amount of $1,595, leaving SALT below the $16,100 standard deduction, so the filer takes the standard deduction plus the $196 non-itemizer charitable deduction. Under that convention, taxable income is $300,150.19 AGI (after the $5,708.82 alimony deduction) minus $16,296, or $283,854.19; regular tax is $68,112.09, and including the $222.64 NIIT gives $68,334.73. The excluded frozen reference instead uses taxable income of $283,059.84 and regular tax of $67,834.07 plus NIIT, giving $68,056.71. The models near the reference took the standard deduction, as the convention does; those that also took the $196 non-itemizer charitable deduction land within $0.14 of $68,334.73 with NIIT or $68,112.09 without it. Others also dropped the alimony deduction or NIIT, used pre-TCJA rules (personal exemptions and miscellaneous itemized deductions, permanently repealed by OBBBA), or subtracted ESI premiums from the stated gross wages. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)." us,scenario_020,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_020,payroll_tax,30,llm_error,thresholds_rates,False,,"The correct employee payroll tax has three parts. Social Security is 6.2% of the 2026 wage base of $184,500, or $11,439. Medicare is 1.45% of the full $300,000 in gross wages, or $4,350. Additional Medicare Tax is 0.9% of the $100,000 above the $200,000 single threshold, or $900, and Texas has no employee state payroll tax. Most wrong answers fall into four groups: using an old or guessed wage base such as $176,100, $181,800 or $183,600; subtracting the $8,389 ESI premiums from Medicare wages; adding the $42,857 FLSA overtime premium on top of gross wages that already include overtime; or working out $16,689 correctly and then submitting a different number." +us,scenario_020,payroll_tax,31,llm_error,payroll_tax_base,False,,"The reference is $16,689. It applies 6.2% Social Security tax up to the 2026 wage base of $184,500 ($11,439), 1.45% Medicare on the full $300,000 ($4,350), and 0.9% Additional Medicare Tax on the $100,000 above the $200,000 single threshold ($900). The listed employer-sponsored insurance premiums do not reduce FICA wages, and the $42,857 FLSA overtime premium is already inside the $300,000 gross wages. The models went wrong in four main ways: they used an old or guessed wage base ($168,600, $176,100, $180,000–$184,200), cut Medicare wages by the $8,389 premium, added the overtime premium on top of gross wages, or worked out the right total and then submitted a different number." us,scenario_020,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident's state income tax before refundable credits is zero. The nationwide aggregation does not create liability in jurisdictions where the household does not reside." us,scenario_021,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_021,head_medicaid_eligible,6,llm_error,thresholds_rates,False,,"At 85 and on Medicare, the head is not in the MAGI adult expansion group, which covers ages 19–64. The only route left is Missouri's aged/blind/disabled (ABD) pathway, and its income test is 85% of the federal poverty level (FPL). For that test, the full $22,000 of Social Security plus about $5,530 of pension counts as unearned income. After the $20 general disregard, countable income is still about 172% of FPL, roughly double the limit, so medicaid_category comes out as none. Every model treated age and disability, plus assets below the limit, as enough to qualify. None compared the head's actual income with the ABD income limit, and PolicyEngine does not let medical expenses or a spend-down bring countable income down." @@ -158,43 +158,43 @@ us,scenario_021,self_employment_tax,1,parse_contract_failure,missing_output,Fals us,scenario_021,snap,1,llm_error,thresholds_rates,False,,"An elderly or disabled SNAP household is exempt from the gross-income test but must still satisfy the one-person net-income limit. With monthly net income of approximately $2,063 even under the model’s own deductions, the household fails that test, so the calculation stops at ineligibility and produces $0 rather than an allotment." us,scenario_021,state_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"Missouri tax is computed from Missouri taxable income after applicable deductions and the 2026 taxable-income threshold, not by applying a flat rate directly to taxable pension income. After those steps, this household has no Missouri taxable income and therefore owes $0 before refundable credits." us,scenario_021,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_022,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is widowed and has no dependent child, so qualifying-surviving-spouse status does not apply and they file single. As a single filer, they itemize $29,735.52. That total is $8,829 of property tax plus PolicyEngine's ~$6,059 California income tax estimate in SALT, which is well under the OBBBA $40,400 cap, plus $15,393 of charity reduced by the new 0.5%-of-AGI floor. They also get the $910 QBI deduction on rental income and the senior deduction, which phases down 6% above $75,000 MAGI to $3,951.05. That leaves $74,552.59 of taxable income, taxed at 2026 single rates. The wrong answers fall into three main groups: models that used joint surviving-spouse brackets came in too low; single filers that left out the state income tax in SALT, the QBI deduction, or the senior phase-down came in too high; and models that applied pre-TCJA law (personal exemptions, 2%-floor miscellaneous deductions, 15%/25% brackets) were wrong because OBBBA made the TCJA rules permanent. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)." +us,scenario_022,federal_income_tax_before_refundable_credits,48,prompt_ambiguity,taxable_income_or_deductions,False,,"The head has no dependent child, so qualifying-surviving-spouse status is unavailable and the 2026 single schedule applies (10% to $12,400, 12% to $50,400, then 22%). With single status, itemizing is the better choice. The prompt lists no state income tax withheld or paid. The frozen reference's SALT adds the engine's $6,058.85 California withholding estimate on federal AGI to $8,829 of real estate tax, totaling $14,887.85 under the $40,400 cap. With charity after the 0.5%-of-AGI floor, itemized deductions are $29,735.52; the phased senior and $910 QBI deductions produce the frozen $74,552.59 taxable income and $11,113.57 tax. These totals depend on unlisted estimated withholding, and the output is excluded from scoring. The exclusion records $11,895.23 when SALT uses the engine's $2,505.87 California liability on policyengine-us 2.15.17, $12,013.38 with the corrected $1,968.81 liability, and $12,192.29 with no state income tax paid ($12,234.66 without the unlisted local sales tax estimate). Models used joint QSS treatment, the old SALT cap or pre-TCJA rules, or omitted deductions; their own California tax estimates or no state tax payment should not be faulted against the unlisted withholding estimate. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)." us,scenario_022,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_022,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Age 77 establishes an aged status but does not itself confer Medicaid eligibility, and Medicare eligibility or assumed enrollment is not a Medicaid eligibility pathway. The head receives no SSI and qualifies under none of California's Medicaid categories, so the aged status cannot overcome the absence of a qualifying pathway." us,scenario_022,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_022,payroll_tax,40,llm_error,state_local_rule,False,,"Every model got the federal part right: 6.2% Social Security ($3,263.37) plus 1.45% Medicare ($763.21) on $52,635 of wages, with no Additional Medicare Tax. The missing step is California's mandatory employee State Disability Insurance (SDI) contribution. For 2026 it is 1.3% of all wages, with no wage cap, which is $684.26 and brings the total to $4,710.83. The wrong answers fall into three groups: (1) left out SDI entirely and stopped near $4,026.58; (2) used an old SDI rate, either 2025's 1.2% (giving $4,658.20), 2024's 1.1% (giving $4,605.57) or 2023's 0.9% (giving $4,500.30); or (3) made arithmetic or answer-submission errors." +us,scenario_022,payroll_tax,41,llm_error,thresholds_rates,False,,"Every model got federal FICA right: $3,263.37 Social Security plus $763.21 Medicare on $52,635 of wages, with no Additional Medicare Tax. The miss is California's mandatory employee SDI. For 2026 it is 1.3% of all wages, with no wage cap, which gives $684.26 and a total of $4,710.83. Most models used an earlier year's SDI rate: 1.2% from 2025, 1.1% from 2024, or 0.9% from 2023. The rest left SDI out because they said CA has no mandatory employee payroll tax." us,scenario_022,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_022,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The correct path starts by removing the $17,624.75 of taxable Social Security, leaving CA AGI of $91,524.41. California still allows unreimbursed employee business expenses as a miscellaneous itemized deduction above a 2% floor measured on federal AGI ($8,351 − ~$2,183 = $6,167.56), so itemized deductions are $30,389.97 and taxable income is $61,134.44. The head has no qualifying dependent, so the surviving-spouse flag does not unlock joint/QSS treatment: the single schedule applies, which gives $2,274.81, less a $153 personal credit and a $153 senior exemption credit ($306). Most models went wrong in one of four ways: they used joint/QSS brackets with doubled personal credits, left out the employee-expense deduction or measured its floor against CA AGI, dropped the senior exemption credit, or failed to remove Social Security from CA AGI." +us,scenario_022,state_income_tax_before_refundable_credits,48,llm_error,household_unit_or_filing_status,False,,"The head is flagged as a surviving spouse but has no dependent child, so the head files as single. The correct steps are: use the $5,706 single standard deduction, apply the single rate schedule, and take a $306 exemption credit ($153 personal plus $153 senior). California excludes all Social Security, so California AGI is $91,524. California never adopted the TCJA suspension of miscellaneous itemized deductions, so the $8,351 of unreimbursed employee expenses counts after a 2% floor based on federal AGI of $109,149, which leaves $6,168. Total itemized deductions are $30,390, taxable income is $61,134, tax on the single schedule is $2,274.81, and subtracting the $306 credit gives $1,968.81. The wrong answers mostly used joint or surviving-spouse brackets, dropped or overstated the employee-expense deduction, or left out the senior exemption credit." us,scenario_022,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_023,federal_income_tax_before_refundable_credits,42,llm_error,other,False,,"AGI is $22,534 ($17,443 wages − $2,778 traditional 401(k) + $8,000 taxable 403(b) distributions − $130 IRA deduction). The OBBBA 2026 single standard deduction of $16,100 leaves $6,434 of taxable income, taxed at 10% for $643. No nonrefundable credit applies. The Saver's Credit is zero because IRC §25B(d)(2) reduces qualified contributions by distributions received in the testing period, and the $8,000 403(b) distribution exceeds the $3,599 of total contributions. The AOTC is zero because no qualified tuition is listed. Most models zeroed out the tax with a phantom Saver's Credit or AOTC. The rest used a stale or pre-OBBBA standard deduction (including the TCJA-expiration personal exemption), wrongly subtracted employer health premiums, or dropped the $8,000 distribution from income." +us,scenario_023,federal_income_tax_before_refundable_credits,43,llm_error,other,False,,"The base computation is AGI $22,534 (wages minus the traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction). Subtracting the 2026 OBBBA single standard deduction of $16,100 leaves $6,434 of taxable income, all taxed at 10%, for $643.43. No nonrefundable credit reduces that amount. The Saver's Credit base is zero because IRC §25B(d)(2) (Form 8880 line 4) subtracts the $8,000 of testing-period 403(b) distributions from the $3,599 of total retirement contributions. The AOTC is zero because no qualified tuition is listed, and the elderly/disabled credit phases out completely at this AGI. Most models wrongly wiped out the tax with a $1,000 Saver's Credit or an AOTC. Others used outdated or TCJA-expiration standard deductions, or excluded the $8,089 ESI premium from wages." us,scenario_023,federal_refundable_credits,20,llm_error,credit_phaseout,False,,"This case has two traps, and a model had to avoid both to reach $0. First, the AOTC eligibility flags do not create a credit. The credit is 100% of the first $2,000 of qualified tuition expenses plus 25% of the next $2,000, and no expenses are listed, so the base is $0 and the refundable 40% portion is also $0. Second, the childless EITC phaseout is measured on the greater of earned income or AGI. AGI is about $22,535: wages of $17,443, minus the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction. That is above the 2026 single childless completion point of about $19,540, so the EITC is fully phased out." us,scenario_023,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"The household qualifies for the FREE meal tier through California’s universal program and categorical eligibility, despite its 1.59 federal-poverty-guideline ratio exceeding the federal income cutoff. The requested output is narrower: it equals 1 only when PolicyEngine returns positive annual free-school-meal support, and this adult-only household has no student receiving meals, so the annual support is zero." us,scenario_023,head_chip_eligible,2,llm_error,categorical_eligibility,False,,"CHIP eligibility requires being in a covered CHIP category and not already eligible for Medicaid; it is not a general adult coverage pathway based solely on falling below an extended income ceiling. The 28-year-old head qualifies for Medicaid through the WORKING_DISABLED_BUY_IN category, which places them in CHIP category 3 and categorically excludes them from CHIP." -us,scenario_023,head_medicaid_eligible,25,prompt_ambiguity,categorical_eligibility,False,,"The head is disabled and has a job, so PolicyEngine places them in the working-disabled buy-in category. In California this is the 250% Working Disabled Program, which uses a much higher income ceiling than the 138% FPL adult expansion. The head's income is 1.41x FPL and their only asset is $130 in the bank, so they pass both tests. Nearly every model checked only the 138% FPL adult expansion limit and an SSI-linked aged/blind/disabled pathway, saw income just above 138%, and stopped without checking the working-disabled buy-in. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release. Reference depends on an unlisted input; output excluded from scoring (meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require)." +us,scenario_023,head_medicaid_eligible,26,prompt_ambiguity,categorical_eligibility,False,,"The head is disabled and employed (24 hours a week at $14/hour), so the engine places them in the working-disabled buy-in category. California runs this as the 250% FPL Working Disabled Program, and it confers Medicaid eligibility even though MAGI is 1.41x FPL, above the 138% adult-expansion limit. Almost every model tested only the 138% FPL expansion limit and an SSI-linked aged/blind/disabled pathway, then answered not eligible. None of them considered the working-disabled buy-in, which has a much higher income ceiling. Two models gave no answer, and one wrote an explanation saying eligible but submitted 0. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release. Reference depends on an unlisted input; output excluded from scoring (meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require)." us,scenario_023,head_medicare_eligible,4,llm_error,age_disability,False,,"The head is 28, so age-based Medicare eligibility at 65 does not apply. For someone under 65, disability-based Medicare requires 24 months of Social Security Disability Insurance (SSDI) entitlement or an end-stage renal disease diagnosis, not just a disability status. This household has no SSDI income and no ESRD, so the head is not eligible. The wrong models treated the `is_disabled` flag by itself as enough for Medicare." us,scenario_023,head_wic_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_023,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_023,payroll_tax,42,llm_error,state_local_rule,False,,"The full $17,442.65 of wages is the base for every payroll tax: Social Security at 6.2% ($1,081.44) and Medicare at 1.45% ($252.92), with no Additional Medicare Tax. California's mandatory employee SDI is charged on top of that, with no wage cap, and comes to $226.75, which is 1.3% of wages, the 2026 rate. Traditional 401(k) deferrals and employer-sponsored insurance premiums do not reduce this base, and 403(b) distributions are not wages. Most models got the federal FICA part right. They then either left out CA SDI or used an older or wrong SDI rate (0.9%, 1.0%, 1.1%, 1.13%, 1.2%). Some also subtracted the $8,089 ESI premium or the 401(k) deferral from the wage base." +us,scenario_023,payroll_tax,43,llm_error,state_local_rule,False,,"Payroll tax is employee FICA on the full $17,442.65 of wages (6.2% Social Security plus 1.45% Medicare, which comes to $1,334.36) plus California's mandatory employee SDI. For 2026, SDI is 1.3% of all wages with no cap, which is $226.75. The derivation text calls the rate ""1%"", but the $226.75 figure is 1.3%. Neither employer-sponsored insurance premiums nor traditional 401(k) deferrals reduce this wage base. The wrong answers fall into four groups: they left out CA SDI, used an outdated SDI rate (0.9%, 1.0%, 1.1%, 1.13% or 1.2% instead of 1.3%), subtracted the $8,089 ESI premiums or the 401(k) deferral from the payroll wage base, or made arithmetic errors." us,scenario_023,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_023,snap,47,prompt_ambiguity,age_disability,False,,"The main trap is the head's 'is disabled' flag. For SNAP, a household counts as elderly/disabled only if a member receives a disability-based benefit such as SSI or SSDI (7 CFR 271.2). Elderly/disabled status is what removes the excess shelter cap and allows the excess medical deduction. The head receives no such benefit, so the shelter deduction stays capped at $744 and no medical deduction applies. Net income is $2,120.25 − $290.72 earned-income deduction − $209 standard deduction − $744 capped shelter = $876.51, which gives $298 − $262.80 = $35.20/month for January–September and $48.18/month for October–December after the FY2027 uprating, or $461.34 for the year. Most models used an uncapped shelter deduction and got near-maximum benefits. The zero-benefit models missed California's broad-based categorical eligibility, which sets the gross income limit at 200% FPL; $2,120/month passes that limit. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)." +us,scenario_023,snap,48,prompt_ambiguity,age_disability,False,,"For SNAP, the USDA definition of disabled (7 CFR 271.2) requires receiving a qualifying disability-based benefit such as SSI or SSDI. This head has an ""is disabled"" flag but receives no such benefit, so the household is not an elderly/disabled SNAP household: it gets no excess medical deduction, and its excess shelter deduction is capped at $744 a month. Monthly gross income is $2,120.25 ($17,443 in wages plus the $8,000 403(b) distribution, divided by 12). That passes California's 200% FPL broad-based categorical eligibility test. Subtracting the 20% earned-income deduction ($290.72), the $209 standard deduction and the capped $744 shelter deduction leaves net income of about $876.51. The frozen reference gives $298 − $262.80 = $35.20 a month for January–September and $48.18 for October–December under the engine's projected FY2027 parameters, totaling $461.34. The release's c_snap_hold_fy2026 convention holds FY2026 for all 12 months; on this capped reading, the current engine with SNAP rounding fixes gives $34/month ($408/year). Most models gave uncapped shelter and medical deductions because of the disability flag; the rest wrongly zeroed SNAP using a 130% gross income test. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)." us,scenario_023,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to satisfy the aged, blind, or disabled categorical gate before income exclusions or benefit-rate calculations apply. The head's unlisted SSI disability status is false under the prompt instructions, so `is_ssi_aged_blind_disabled=False` and SSI is $0 without an income calculation." -us,scenario_023,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"California AGI is $22,534.34: wages minus the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction. Subtracting the 2025 single standard deduction of $5,706 leaves $16,828.34 of taxable income. Tax at 1% up to $11,079 and 2% above that is $225.78. The nonrefundable credits ($153 personal exemption credit plus the $60 renter's credit) total only $213, so $12.78 remains. Models missed this small positive amount in four main ways: they claimed the credits wiped out the tax, left out the 403(b) distribution, used projected 2026 or older-year parameters instead of the 2025 values, or dropped or invented a nonrefundable credit." -us,scenario_023,state_refundable_credits,46,llm_error,credit_phaseout,False,,"California's CalEITC covers childless filers aged 18 and over with earnings up to about $32k in 2026. It does not depend on qualifying for the federal EITC or having children, and it is refundable. In the phase-out range, the credit is computed on the greater of earned income ($17,442.65) or AGI ($22,534.34). This filer's AGI is higher because it includes $8,000 of taxable 403(b) distributions, minus the 401(k) deferral and IRA contribution, so the credit is computed on AGI and comes to $95.15. Models that answered $0 wrongly denied childless or non-federal-EITC eligibility, or thought the credit had already phased out. Models that answered higher amounts ran the phase-out on wages alone, guessed at the credit schedule, or added nonrefundable credits." +us,scenario_023,state_income_tax_before_refundable_credits,48,llm_error,thresholds_rates,False,,"California AGI is $22,534.34: wages minus the 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the IRA deduction. Subtracting the $5,706 standard deduction leaves $16,828.34 of taxable income. At 1% up to $11,079 and 2% above that, the tax is $225.78. The nonrefundable credits total $213 ($153 personal exemption credit and $60 renter's credit), so they fall $12.78 short of the tax. Most models went wrong in one of four ways: they assumed the credits wipe out a low-income filer's tax, they dropped the 403(b) distribution or the 401(k)/IRA adjustments, they left out one of the two credits or made up a disability credit, or they used stale or projected parameters instead of the 2025 amounts that the reference carries into 2026." +us,scenario_023,state_refundable_credits,47,llm_error,credit_phaseout,False,,"California's CalEITC is refundable and open to childless filers aged 18 or older. Its schedule is separate from the federal EITC, and its long phase-out tail runs past $30,000 of income. Because this filer's AGI ($22,534, which includes $8,000 of taxable 403(b) distributions and subtracts the 401(k) and IRA deferrals) is higher than earned income ($17,442.65), the phase-out uses AGI, which leaves a CalEITC of $95.15. The models that answered $0 wrongly treated the filer as ineligible: they said a child was required, tied CalEITC to federal EITC, treated pension income as disqualifying investment income, or placed the cutoff too low. Most of the positive answers worked the phase-out from wages alone or guessed the tail amount loosely." us,scenario_023,tanf,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_025,federal_income_tax_before_refundable_credits,31,llm_error,thresholds_rates,False,,"The OBBBA made the TCJA rate structure permanent, so this case only needs three 2026 MFJ parameters: the $32,200 standard deduction, the 10% bracket ending at $24,800, and the 12% rate after that. AGI is $94,925 (spouse wages plus taxable pension). Taxable income is $62,725, so the tax is $2,480 + 12% × $37,925 = $7,031.04, with no nonrefundable credits. Most models missed for one of three reasons: they assumed the TCJA sunset (personal exemptions, a smaller standard deduction and a 15% bracket); they used 2025 or misestimated 2026 thresholds; or they reached about $7,031 and then submitted a different, unsupported final number." us,scenario_025,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_025,head_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 in the absence of a qualifying disability, ESRD, or other explicitly listed pathway. The 61-year-old head has no such condition, so the age test yields 0 (No)." us,scenario_025,payroll_tax,8,llm_error,payroll_tax_base,False,,"The spouse's $62,725 of wages are the only FICA-covered earnings. They are taxed at 6.2% for Social Security and 1.45% for Medicare, which gives $4,798.48 ($3,888.97 + $909.52). The head has no wages. The $32,200 pension is outside the FICA base. The head's ESI premium is not subtracted from the spouse's wages. Ohio has no employee state payroll tax, and Additional Medicare Tax does not apply. Several models got this computation right but submitted a different number. Others added Ohio income tax, subtracted the ESI premium, used the combined employer-plus-employee rate, or counted wages for the head." -us,scenario_025,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"The reference has four parts. First, Ohio AGI is federal AGI ($94,925.30) minus the unreimbursed medical care deduction. That deduction is $180.60: $6,500 of premiums plus $800 of other medical expenses, less the 7.5%-of-FAGI floor. The premiums count only above the floor because the head can get employer-paid coverage. Second, two $1,900 exemptions apply because Ohio MAGI is over $80,000. Third, the 2026 rate schedule adds a base amount of about $332 to 2.75% of income over $26,050. Fourth, the $200 retirement income credit applies for the $32,200 pension. The most common wrong answer, $1,589.56, gets the exemptions and the credit right but leaves out the $332 base amount and the medical deduction. Most other answers also use the wrong exemption amount, use the 3%+ brackets from before 2024, subtract a federal standard deduction, or add or drop credits." +us,scenario_025,state_income_tax_before_refundable_credits,48,llm_error,thresholds_rates,False,,"The reference has five steps. First, Ohio's unreimbursed-medical deduction: the $6,500 of premiums plus the $800 of other medical expenses above 7.5% of FAGI gives $180.60, so OAGI is $94,744.70. Second, two $1,900 exemptions (the over-$80k tier). Third, Ohio's schedule taxes taxable income of $90,944.70 at a $332 base amount plus 2.75% of the excess over $26,050, giving $2,116.61. Fourth, the $200 retirement income credit for the spouse's $32,200 pension brings the tax to $1,916.61. Fifth, no joint filing credit applies because the head has no income. The most common wrong answer, $1,589.56, gets the exemptions and the credit right. It drops the $332 base amount and skips the medical deduction. Other answers err on exemption amounts, outdated brackets, an invented standard deduction, or by counting the $21,208 employer premiums as income adjustments or deductions." us,scenario_026,child1_chip_eligible,5,llm_error,categorical_eligibility,False,,"NC Health Choice merged into NC Medicaid in 2023. North Carolina now covers school-age children up to 211% FPL, plus a 5% disregard, as Medicaid (Medicaid-expansion CHIP) under the OLDER_CHILD category, not as a separate CHIP program. PolicyEngine finds the 11-year-old Medicaid-eligible at this household income. Because CHIP covers only children who fail Medicaid, that Medicaid eligibility makes the child CHIP-ineligible. Every model that answered treated the 211% band as a separate CHIP program sitting above a lower Medicaid limit. Each one called the child CHIP-eligible instead of routing them to Medicaid first and excluding CHIP." -us,scenario_026,child1_medicaid_eligible,46,llm_error,thresholds_rates,False,,"North Carolina folded NC Health Choice (separate CHIP) into Medicaid. Children ages 6–18 are now covered in the Medicaid older-child category up to 211% FPL, or 216% with the 5-percentage-point MAGI disregard. The household's MAGI is $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA deductions, or $82,301. That is 2.13× the 2026 FPL for five people, which is under the limit, so the 11-year-old qualifies. Most models used the old 133%/138% FPL older-child Medicaid limit. Others overstated income by double-counting overtime or using gross wages, used the wrong poverty guideline, or wrongly treated employer-sponsored insurance as a Medicaid bar. That ESI bar applies only to CHIP, not to Medicaid." +us,scenario_026,child1_medicaid_eligible,47,llm_error,thresholds_rates,False,,"Household MAGI is the spouse's $85,209 in wages, which already include overtime, minus the $2,778 traditional 401(k) contribution and the $130 traditional IRA deduction. That gives $82,301, or 2.13x the 2026 FPL for a family of five. In April 2023, North Carolina moved NC Health Choice into Medicaid, so children aged 6-18 now qualify for Medicaid up to 211% FPL, or 216% with the 5-point MAGI disregard. A household at 2.13x FPL falls under that limit. Most models used the old 133%/138% Medicaid limit or treated 211% as a hard cap without the disregard, and some inflated MAGI or wrongly treated employer-sponsored insurance as a bar to Medicaid." us,scenario_026,child2_chip_eligible,5,llm_error,categorical_eligibility,False,,"Child 2 is 11. In the engine, North Carolina's Medicaid OLDER_CHILD category covers this child at the household's income, so the child is Medicaid-eligible. CHIP only covers children who do not qualify for Medicaid, which makes child2_chip_eligible = No. NC folded NC Health Choice into Medicaid, so the income band that the models called ""CHIP, above Medicaid and up to 211-216% FPL"" is Medicaid coverage for a school-age child. All four models that answered found income below the NC children's limit but assigned the child to CHIP instead of Medicaid, and never applied the rule that Medicaid eligibility excludes CHIP." -us,scenario_026,child2_medicaid_eligible,46,llm_error,thresholds_rates,False,,"MAGI is the spouse's $85,209 in wages minus the traditional 401(k) and IRA deductions, about $82,300. Against the 2026 FPL for a household of five (about $38,700), that is 2.13 times FPL. NC Health Choice was folded into Medicaid in April 2023, so NC Medicaid now covers children ages 6–18 up to 211% FPL, or 216% after the 5-point MAGI disregard, and 2.13 falls under that limit. The models got this wrong in three main ways: they used the old 133% Medicaid line for ages 6–18 (when 133–211% went to separate CHIP), they compared against 210–211% without the 5-point disregard or measured income against an older, lower FPL, or they treated employer-sponsored insurance as a bar to Medicaid." +us,scenario_026,child2_medicaid_eligible,47,llm_error,thresholds_rates,False,,"North Carolina folded NC Health Choice (its separate CHIP) into Medicaid in 2023. Since then, the Medicaid limit for older children (ages 6–18) has been 211% FPL, or 216% once the 5-point MAGI disregard is applied, not the old 133%/138% floor. The household's MAGI is 2.13× FPL: the spouse's wages minus the traditional 401(k) and IRA deductions, measured against the 2026 FPL for 5. That is under the limit, so the 11-year-old falls in the OLDER_CHILD Medicaid category and is eligible. Most models applied the outdated 133% Medicaid limit and treated 133–211% as CHIP. Others overstated the FPL ratio by using gross wages or an old poverty guideline, left out the 5% disregard, or wrongly treated employer-sponsored insurance as a bar to Medicaid." us,scenario_026,child3_chip_eligible,7,llm_error,categorical_eligibility,False,,"Every model placed 9-year-old Child 3 in a CHIP band that sits above Medicaid, using the old 133%-to-211% FPL NC Health Choice structure. North Carolina moved NC Health Choice into Medicaid in 2023, and PolicyEngine finds Child 3 Medicaid-eligible under the OLDER_CHILD category. A Medicaid-eligible child is excluded from CHIP by construction, so the answer is No however each model computed MAGI. The decisive step is the Medicaid-first ordering with North Carolina's older-child Medicaid limit, not any CHIP income ceiling." -us,scenario_026,child3_medicaid_eligible,46,llm_error,thresholds_rates,False,,"NC Health Choice, the state's separate CHIP, was folded into Medicaid in April 2023. Since then, NC Medicaid covers children ages 6-18 up to 211% FPL, or about 216% with the 5% MAGI disregard. The household's MAGI is about $82,301: spouse wages of $85,209, which already include overtime, minus the $2,778 traditional 401(k) and $130 traditional IRA deductions. That is 2.13 times the 2026 FPL for five people, so Child 3 qualifies in the OLDER_CHILD category. Most models used the old 133%/138% limit and assumed incomes above it go to a separate CHIP. Others compared gross wages instead of MAGI, left out the disregard, used the wrong FPL, or wrongly treated employer coverage as a bar to Medicaid." +us,scenario_026,child3_medicaid_eligible,47,llm_error,thresholds_rates,False,,"When NC Health Choice moved into Medicaid in 2023, North Carolina's Medicaid limit for older children (ages 6-18) rose to 211% FPL, or 216% after the 5% MAGI disregard. It is no longer 133%/138%. Household MAGI is $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA contributions, which is about $82,301. Against the 2026 five-person FPL of about $38,680, that is 2.13x FPL, under the 216% limit, so the 9-year-old is Medicaid-eligible in the OLDER_CHILD category. Most models used the old 133% limit and treated the 133-211% band as separate CHIP. Others compared gross wages instead of MAGI, left out the 5% disregard, used a wrong FPL, or wrongly treated employer coverage as a bar to Medicaid." us,scenario_026,federal_income_tax_before_refundable_credits,22,llm_error,taxable_income_or_deductions,False,,"2026 law keeps the TCJA structure — personal exemptions repealed, a $32,200 MFJ standard deduction — and adds the deduction for qualified FLSA overtime premiums, so the $7,746.26 premium comes out of taxable income rather than being stacked onto wages that already include it, leaving $42,354.26 taxable and $4,586.51 of tentative tax. The child tax credit is $2,200 per qualifying child in 2026, and its nonrefundable portion offsets the entire liability first, with the $1,700-per-child ACTC as the residual, so $6,600 of credit for three children drives the answer to exactly $0. The wrong answers split into two clusters: models that applied the expired pre-TCJA regime (a reverted ~$16,600 standard deduction, ~$5,200 personal exemptions, a 15% bracket, a $1,000-per-child credit), and models that got the deduction stack roughly right but capped the nonrefundable CTC at $500–$1,100 per child by treating the refundable ACTC as consumed first. Every wrong model ignored the qualified-overtime deduction, and three double-counted the $7,746 premium on top of the $85,209 gross wage figure." -us,scenario_026,federal_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"AGI is $82,301 ($85,209 wages minus the $2,778 traditional 401(k) and $130 traditional IRA). Taxable income is about $42,355 after the $32,200 2026 MFJ standard deduction and the OBBBA qualified overtime deduction for the $7,746 FLSA overtime premium. That makes pre-credit tax about $4,586.51. Under OBBBA, the CTC is $2,200 per child, or $6,600, so $2,013.49 of credit is left unused after offsetting tax. That amount is refundable because it falls below the $1,700-per-child cap and the 15%-of-earnings phase-in limit, and EITC is $0. Most models missed the overtime deduction, used the outdated $2,000 or sunset $1,000 per-child CTC, or wrongly claimed that the tax fully absorbed the credit." +us,scenario_026,federal_refundable_credits,41,llm_error,taxable_income_or_deductions,False,,"AGI is $82,301: $85,209 in wages, minus the $2,778 traditional 401(k) deferral and the $130 traditional IRA deduction. From that, the 2026 MFJ standard deduction of $32,200 and the OBBBA qualified-overtime deduction for the $7,746 FLSA overtime premium are subtracted, giving taxable income of about $42,355 and pre-credit tax of about $4,586.51. The OBBBA CTC of $2,200 × 3 = $6,600 cuts that tax to zero. The unused $2,013.49 is fully refundable as ACTC because it is under both the $1,700-per-child cap ($5,100) and the 15%-of-earnings-over-$2,500 limit. EITC is zero at this income. Most wrong answers overstated pre-credit tax by skipping the overtime deduction, which shrinks or erases the refundable remainder. Others used the expired $1,000 credit or the pre-OBBBA $2,000 credit, or ignored the rule that ACTC is capped at the unused CTC." us,scenario_026,free_school_meals_eligible,1,llm_error,thresholds_rates,False,,"Free school meals require income within the free-meal limit or categorical eligibility through a qualifying program. This household's countable income of $85,208.82 equals 220% of the $38,680 federal poverty guideline, exceeds even the 185% reduced-price limit, and has no categorical eligibility, so its school-meal tier is paid." us,scenario_026,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_026,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -203,7 +203,7 @@ us,scenario_026,reduced_price_school_meals_eligible,1,llm_error,categorical_elig us,scenario_026,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_026,state_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"North Carolina starts from federal AGI of $82,300.52. That is the spouse's $85,209 in wages, which already include the overtime premium, minus the traditional 401(k) and IRA contributions. NC then subtracts two deductions: the $25,500 MFJ standard deduction and the income-tiered NC child deduction. For MFJ filers with AGI between $80,001 and $100,000 the child deduction is $1,500 per child, or $4,500 for three children, which leaves $52,300.52 taxed at the flat 3.99% rate. Most models left out the child deduction or used the wrong income tier. Others got AGI wrong: some counted the FLSA overtime twice, some subtracted it, some subtracted the head's ESI premiums, and some ignored the retirement contributions. A few used the wrong rate." +us,scenario_026,state_income_tax_before_refundable_credits,37,llm_error,state_local_rule,False,,"North Carolina starts from federal AGI. Here that is about $82,301: $85,209 in wages, which already include the $7,746 overtime premium, minus the $2,778 traditional 401(k) and the $130 traditional IRA. From that, NC subtracts the $25,500 joint standard deduction and the income-tiered child deduction. For joint filers with AGI between $80,001 and $100,000, the child deduction is $1,500 per child, or $4,500 for three children. Tax is 3.99% of $52,300.52. Most models either left out the child deduction or used the wrong AGI tier ($500, $1,000, $2,000, $2,500 or $3,000 per child). Others got AGI wrong by adding the overtime premium on top of wages, subtracting it, or subtracting ESI premiums, or they used an outdated NC rate (4.25%, 4.5% or 4.75%)." us,scenario_026,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,federal_income_tax_before_refundable_credits,2,llm_error,other,False,,"The household has no positive federal income tax before refundable credits because taxable income and every additional component of that output are zero. A missing submission is a parse-contract failure, while a negative submission contradicts both the model's own zero-tax derivation and the output's nonnegative liability construction." us,scenario_027,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -216,7 +216,7 @@ us,scenario_027,local_income_tax,1,parse_contract_failure,missing_output,False,, us,scenario_027,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_027,snap,46,llm_error,categorical_eligibility,False,,"Connecticut uses broad-based categorical eligibility (BBCE) through TANF non-cash benefits. Gross income of $2,536.25/month is about 1.44× poverty, below the BBCE limit, so the household is categorically eligible. That removes the federal asset test (the $48,000 doesn't count against it) and the federal 100% FPL net income test. Under the regular formula, the expected contribution is $636 (30% of $2,118 net income), which exceeds the $546 two-person maximum, so the formula benefit is $0. But eligible one- and two-person households get the minimum allotment: 8% of the FY2026 one-person maximum of $298, which is $24/month, or $288/year. Most models applied the $4,250–$4,500 elderly/disabled asset limit or the net income test and returned $0. Others recognized categorical eligibility but left out the minimum allotment or used FY2025's $23 minimum." +us,scenario_027,snap,47,llm_error,categorical_eligibility,False,,"Connecticut uses broad-based categorical eligibility through TANF non-cash benefits. Because gross income is $2,536.25 a month, about 1.44 times the poverty guideline and under the 200% limit, the household is categorically eligible. That eligibility waives both the asset test (so the $48,000 doesn't matter) and the 100% FPL net income test. Net income is $2,118 a month, so the expected contribution of $636 is larger than the $546 two-person maximum and the formula benefit is zero. A categorically eligible one- or two-person household still gets the FY2026 minimum allotment of $24 a month, which is $288 a year. Most models either applied the asset limit or the net income test that BBCE waives, or computed a zero formula benefit and never applied the minimum allotment. A few used the old $23 minimum." us,scenario_027,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,spouse_medicaid_eligible,13,llm_error,categorical_eligibility,False,,"The spouse, 39, qualifies through no Medicaid category in the engine. For the adult expansion group, Medicaid MAGI adds back all non-taxable Social Security benefits, so the head's $30,160 SSDI plus $275 interest gives household MAGI of $30,435. That is 1.41x FPL, above the 138% limit. Being blind and disabled confers no separate category either: the spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources far exceed the SSI couple limits. Models said yes by either dropping Social Security from MAGI or treating blindness and disability as enough on their own under an aged, blind and disabled (ABD, HUSKY C) pathway." us,scenario_027,spouse_medicare_eligible,12,llm_error,age_disability,False,,"Medicare eligibility requires age 65+, ESRD/ALS, or 24 months of the person's own Social Security disability benefit receipt. A disability flag alone does not qualify anyone. The 39-year-old spouse is flagged blind and disabled but has $0 Social Security disability income of their own; the $30,160 SSDI belongs to the head. The spouse therefore has no Medicare pathway. The models mistook the disability flag for SSDI entitlement, and some credited the head's SSDI to the spouse." @@ -244,7 +244,7 @@ us,scenario_028,child3_medicaid_eligible,6,llm_error,categorical_eligibility,Fal us,scenario_028,child3_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five, so a 10-year-old fails the categorical age requirement. The household also has approximately $61,277 of income, exceeding the applicable 185%-of-poverty WIC income limit; Child 3 therefore fails both the categorical and income tests." us,scenario_028,federal_income_tax_before_refundable_credits,20,llm_error,credit_phaseout,False,,"Under 2026 law (OBBBA made the TCJA structure permanent), the HoH filer has AGI of $60,010. Child support is not taxable. The $24,150 standard deduction leaves $35,860 of taxable income and $3,949.20 of tax. The full $2,200-per-child CTC ($6,600) is applied first as a nonrefundable credit up to that liability, which zeroes it out. Only the leftover $2,650.80 is refundable ACTC. The wrong answers come from three mistakes: assuming TCJA expired (personal exemptions and a $1,000 CTC), carving the refundable ACTC out of the CTC before applying the nonrefundable portion, or applying only part of the CTC against the liability." -us,scenario_028,federal_refundable_credits,42,llm_error,credit_phaseout,False,,"The reference has two parts. The first is the refundable CTC: OBBBA's 2026 CTC is $2,200 per child, so $6,600 for three children. AGI is $60,010; after the $24,150 HoH standard deduction, taxable income is $35,860 and pre-credit tax is $3,949.20. The unused $2,650.80 is refundable because it is below the $5,100 cap ($1,700 per child) and below the 15% earned-income limit. The second part is the EITC, which has not fully phased out at $60,010: $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. Most wrong answers fall into one of a few errors: using the pre-OBBBA $2,000 or post-TCJA $1,000 per-child credit, paying the full $1,700/child cap without limiting it to unused CTC, saying the CTC was fully absorbed by tax, dropping the EITC by assuming the 3-child phaseout ends below $60,010, or using the wrong 2026 EITC maximum or phaseout threshold." +us,scenario_028,federal_refundable_credits,43,llm_error,credit_phaseout,False,,"The reference has two parts. The first is a refundable CTC of $2,650.80: the 2026 CTC is $2,200 per child, or $6,600 for three children. Pre-credit tax is $3,949.20, from the $24,150 HoH standard deduction and the 10%/12% brackets on $35,860 of taxable income. The unused $2,650.80 sits below both the $5,100 cap ($1,700 per child) and the $8,625 earned-income limit, so the unused amount is what binds. The second part is an EITC of $624.13: $8,231 − 21.06% × ($60,010 − $23,890), which stays positive because the three-child phase-out ends near $62,970. Most wrong answers left out the EITC as 'phased out', used the old $2,000 (or $1,000) per-child CTC, skipped the unused-CTC limit by paying the full $1,700 per child, or used a slightly wrong EITC maximum or phase-out threshold." us,scenario_028,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -257,14 +257,14 @@ us,scenario_028,self_employment_tax,1,parse_contract_failure,missing_output,Fals us,scenario_028,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,state_income_tax_before_refundable_credits,11,llm_error,taxable_income_or_deductions,False,,"Pennsylvania's personal income tax is a flat 3.07% on the gross amount of each taxable income class. PA has no standard deduction and no personal or dependent exemptions, and child support is not taxable. The correct tax is 3.07% × ($60,000 wages + $10 interest) = $1,842.31. The household gets no tax forgiveness: eligibility income of about $61,277 (child support included) is far above the $35,000 full-forgiveness limit for an unmarried claimant with three dependents ($6,500 + 3 × $9,500), and partial forgiveness ends about $2,250 above that limit. The wrong answers came from invented deductions or exemptions, invented credits or forgiveness, arithmetic slips, or a submitted value that did not match the model's own correct reasoning." -us,scenario_028,state_refundable_credits,44,llm_error,state_local_rule,False,,"Pennsylvania's FY2025-26 budget created the Working Pennsylvanians Tax Credit. It is a refundable state EITC equal to 10% of the federal EITC, effective from tax year 2025. This head of household has three children and about $60,010 of AGI, which sits near the end of the 2026 federal EITC phase-out, so the federal EITC is $624.13 and the PA refundable credit is $62.41. Nearly every model relied on the outdated rule that PA has no refundable credits, or mistook Tax Forgiveness or the child and dependent care credit for the only relevant programs, and answered $0. One model applied the 10% rule correctly but used an imprecise federal EITC." +us,scenario_028,state_refundable_credits,45,llm_error,state_local_rule,False,,"Pennsylvania's 2025 budget created the Working Pennsylvanians Tax Credit, a refundable state EITC equal to 10% of the federal EITC, starting in tax year 2025. This head of household has three qualifying children and about $60,010 of AGI. That is still inside the federal three-child EITC phase-out range, so the federal EITC is $624.13 and the PA credit is $62.41. Nearly every model treated PA as having no refundable credit, or only thought about Tax Forgiveness and the child-care credit. One model used the right 10% rule but started from a rounded federal EITC." us,scenario_028,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_029,federal_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"The only AGI item in this household is $312 of taxable interest: ""financial assistance"" is a non-taxable assistance receipt in PolicyEngine's income taxonomy and feeds no AGI component, so the $50,000 never reaches the 1040. With AGI of $312 against a 2026 single standard deduction of roughly $16,100 under the permanent OBBBA rules, taxable income is $0 and federal income tax before refundable credits is $0. All three models converted the $50,000 assistance figure into taxable income (wages or other ordinary income) and then ran a full bracket computation on roughly $50,312 of AGI, which is the single step that separates every wrong answer from the reference." us,scenario_029,head_medicaid_eligible,20,llm_error,taxable_income_or_deductions,False,,"Oklahoma expanded Medicaid in 2021, so a 21-year-old non-dependent adult qualifies in the ACA adult group if MAGI is at or below 138% FPL. The $50,000 of financial assistance is a non-taxable transfer. It is not part of AGI or MAGI, so MAGI is only the $312 of taxable interest, about 0.02x FPL. MAGI categories have no asset test, and having employer-sponsored insurance does not block Medicaid eligibility. Most models wrongly added the $50,000 to income. The rest wrongly said Oklahoma never expanded Medicaid, required a disability, pregnancy or parent pathway, or applied asset limits." us,scenario_029,payroll_tax,1,llm_error,payroll_tax_base,False,,"The household lists no wages at all: its only income items are $312 of taxable interest and $50,000 of financial assistance, alongside assets and expenses. Employee Social Security and Medicare taxes attach exclusively to wages and salaries under FICA (IRC §3101, on ""wages"" as defined in §3121(a)); interest is unearned investment income and financial assistance is a non-employment transfer, and neither enters the FICA wage base. With a zero wage base, employee OASDI, employee HI, and Additional Medicare Tax are each $0, giving a household employee-side payroll tax of $0." us,scenario_029,snap,1,llm_error,asset_resource,False,,"SNAP eligibility requires satisfying Oklahoma’s applicable resource test before the benefit formula is applied. This one-person household has $4,250 in bank assets and $50,735 in stock assets, so it fails the resource limit and receives no SNAP benefit regardless of its low interest income or the maximum allotment." us,scenario_029,state_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"The only taxable income in this household is $312 of taxable interest; the $50,000 of ""financial assistance"" is a non-taxable transfer that never enters federal AGI, and Oklahoma's income tax starts from federal AGI, while the $4,250 bank balance and $50,735 of stock are assets that generate no income under the ""treat any unlisted numeric input as 0"" instruction. Against Oklahoma's single standard deduction plus the $1,000 personal exemption, $312 of interest leaves zero Oklahoma taxable income, so ok_income_tax_before_refundable_credits — and therefore the 51-jurisdiction sum — is $0. Every wrong answer is produced by the same step: adding the $50,000 financial assistance into Oklahoma AGI and then applying the graduated OK rate schedule to roughly $37,000–$43,000 of invented taxable income." -us,scenario_029,state_refundable_credits,45,llm_error,categorical_eligibility,False,,"Oklahoma's refundable Sales Tax Relief Credit (ok_stc) pays $40 per personal exemption to any adult filer whose gross household income is at or below $20,000. It does not depend on earned income, dependents, age 65+, disability, or tax liability; those factors only raise the limit to $50,000. PolicyEngine counts gross income from the household's income sources, which here is only the $312 of taxable interest. The $50,000 of financial assistance is not counted, so this single 21-year-old filer with one exemption receives $40. Most models looked only at the EITC, saw no earned income, and stopped. A smaller group named the credit but added the $50,000 of financial assistance to gross income, which pushed the filer over the $20,000 limit." +us,scenario_029,state_refundable_credits,46,llm_error,categorical_eligibility,False,,"The $40 is Oklahoma's refundable Sales Tax Relief Credit (ok_stc): $40 per exemption for a single filer whose gross income is at or below $20,000. PolicyEngine measures that income as $312, which is the taxable interest alone, and does not add the $50,000 of financial assistance. One exemption times $40 gives $40. The credit is refundable whether or not the filer owes any tax, and it does not depend on earned income, dependents, age 65+, or disability. Every model answered $0. Most never considered the sales tax credit and only checked the EITC. Others recognized the credit but wrongly added the $50,000 of financial assistance to gross income, which pushed the filer over the $20,000 limit." us,scenario_030,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Only the $13,000 of wages enters adjusted gross income in this household. The listed educational assistance and financial assistance do not add $17,000 of taxable income, so AGI remains below the 2026 single-filer standard deduction and taxable income and pre-refundable-credit federal income tax are both zero." us,scenario_030,federal_refundable_credits,12,llm_error,categorical_eligibility,False,,"A childless EITC claimant in 2026 must satisfy the minimum-age requirement, and this 23-year-old does not. Pregnancy does not create a qualifying child for EITC or CTC purposes, so every evaluated refundable credit component is zero." us,scenario_030,head_chip_eligible,5,llm_error,categorical_eligibility,False,,The head qualifies for Medicaid under the pregnant-person category. Medicaid eligibility precludes CHIP eligibility under PolicyEngine because CHIP covers qualifying children who do not qualify for Medicaid; the Texas CHIP perinatal income threshold therefore never determines this output. @@ -273,11 +273,11 @@ us,scenario_030,head_wic_eligible,2,llm_error,household_unit_or_filing_status,Fa us,scenario_030,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_030,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on the $13,000 wage base. Those components are $806.00 and $188.50, totaling $994.50 without whole-dollar rounding." us,scenario_030,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_030,snap,47,llm_error,categorical_eligibility,False,,"The SNAP unit is one person. Countable income is $1,083.33 a month in wages plus $1,000 a month in financial assistance, counted as unearned income. The $5,000 of educational assistance is excluded. Gross income is $2,083.33 a month. That is above the regular 130% FPL limit but below Texas's broad-based categorical eligibility limit of 165% of the FY2026 guideline (165% × $1,304.17 = $2,151.88). Because the household is categorically eligible through TANF non-cash eligibility, the gross and net income tests are waived. The expected contribution ($498 a month) exceeds the $298 maximum allotment, so this one-person household gets the $24 monthly minimum allotment, or $288 a year. Most models either stopped at the 130% gross limit or the net income test, or skipped the minimum allotment. A smaller group left out the $12,000 of financial assistance and calculated a large regular benefit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))." +us,scenario_030,snap,48,llm_error,categorical_eligibility,False,,"Texas grants broad-based categorical eligibility through TANF non-cash benefits, with a 165% FPL gross limit. That waives the federal 130% gross test and the 100% net test. Countable gross income is $2,083.33/month: $1,083.33 in wages plus $1,000 in financial assistance, with educational assistance excluded. That is 1.60× the $1,304.17 guideline, so the household stays eligible. Net income is $1,658, so the expected contribution is $498. That exceeds the $298 one-person maximum, and an eligible 1–2 person household then gets the $24 minimum allotment, or $288 a year. Most models either applied the waived 130%/100% tests and returned $0, or left out the $12,000 financial assistance and got about $100/month. Others counted the pregnancy as a second household member. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))." us,scenario_030,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_030,tanf,2,llm_error,categorical_eligibility,False,,Texas TANF requires a family with a dependent child; pregnancy alone does not turn the unborn child into a qualifying dependent child or create a two-person assistance unit for this annual TANF calculation. The single adult therefore has no qualifying child and receives zero TANF. us,scenario_031,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The $1,165 alimony expense reduces $14,408 of IRA and pension income to $13,243 of adjusted gross income, while Social Security remains nontaxable. The applicable single-filer standard deduction, including the age-65 addition, exceeds that income, producing zero taxable income and therefore zero federal income tax before refundable credits." -us,scenario_031,head_medicaid_eligible,44,llm_error,state_local_rule,False,,"At 67, the head qualifies by age alone for California's Aged & Disabled pathway, the optional senior-or-disabled category; neither SSI receipt nor a disability is needed, and Medicare eligibility bars only the 19–64 expansion group. Income is counted under SSI rules: $23,853.47 gross, with alimony paid not subtracted. California then subtracts a $230/month ($2,760/year) disregard in place of SSI's $20 general exclusion. That leaves $21,093.47, which is under 138% of the 2026 FPL ($22,024.80). Resources of $4,200 are far below California's $130,000 individual limit. Almost every model compared gross income, or income less only the $20/month exclusion, against 138% FPL. The rest denied the aged category, treated Medicare as a bar, or used the $2,000 SSI resource limit." +us,scenario_031,head_medicaid_eligible,45,llm_error,state_local_rule,False,,"The head is 67, so they qualify for California's aged/disabled (senior-or-disabled) pathway on age alone. SSI receipt and disability are not required, and Medicare eligibility bars only the 19–64 expansion group. Under SSI income-counting rules, Social Security, pension and IRA income are all counted in full, for a gross of $23,853.47. California's $230 monthly disregard ($2,760 a year) replaces SSI's $20 general exclusion and brings countable income to $21,093.47, under the 2026 limit of 138% FPL ($22,024.80). Assets of $4,200 are far below the $130,000 limit. Almost every model either compared gross income to the limit, applied only the $20 exclusion, used a stale or understated FPL, used the old $2,000 asset limit, or wrongly assumed that Medicare age rules out Medicaid." us,scenario_031,snap,6,llm_error,categorical_eligibility,False,,"The head is 67, so this is an elderly household. PolicyEngine does not let California's 200% FPL broad-based categorical eligibility skip the net income test for this household. Monthly gross income is $1,987.75 (Social Security, pension and IRA). The only deduction is the $209 standard deduction. There are no shelter costs, and the $50/yr OTC medical expense is below the $35/mo threshold, so net income is about $1,779/mo. That is well above the 1-person net limit of about $1,305/mo (100% FPL), so the household is not SNAP-eligible. The one/two-person minimum benefit only applies to eligible households, so SNAP is $0. Five models treated BBCE as automatic eligibility and paid the minimum benefit. The sixth undercounted income." us,scenario_031,state_income_tax_before_refundable_credits,1,llm_error,credit_phaseout,False,,"California first computes $75.37 of tax on $7,537 of taxable income, then applies $312.93 of nonrefundable credits, including the aged-or-blind exemption credit. Those credits fully eliminate the liability, so state income tax after nonrefundable credits and before refundable credits is $0." us,scenario_032,child1_chip_eligible,13,llm_error,health_coverage,False,,"CHIP is residual by statute: 42 U.S.C. 1397jj(b)(1)(A) and 42 CFR 457.310 exclude from ""targeted low-income child"" any child who is eligible for Medicaid, so the CHIP question is settled by first running the Medicaid test. Minnesota's Medical Assistance covers children ages 6-18 — PolicyEngine's OLDER_CHILD category — far above this household's $42,664 for three people (~160% FPL), so the child is Medicaid-eligible and thereby barred from CHIP. Every wrong model instead applied a standalone CHIP income ceiling, most of them quoting Minnesota's 275%/283% FPL child limits, which are the Medical Assistance and MinnesotaCare bands rather than CHIP bands. They read ""income below the child-coverage limit"" as qualifying when it is precisely what disqualifies: several stated the Medicaid fact outright and then used it backwards." @@ -287,14 +287,14 @@ us,scenario_032,child1_medicaid_eligible,4,llm_error,categorical_eligibility,Fal us,scenario_032,child1_medicare_eligible,2,llm_error,categorical_eligibility,False,,"Medicare eligibility is determined separately for each person and does not pass from a Social Security beneficiary to a dependent. The six-year-old child is under age 65, and the stated facts include no end-stage renal disease, ALS, disability-based Medicare entitlement, or other qualifying pathway, so the child's eligibility is No." us,scenario_032,child1_wic_eligible,6,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday; a six-year-old is outside the federally eligible categorical group. Income eligibility cannot override this age restriction, so the child is ineligible regardless of whether household income falls below 185% of the federal poverty guidelines." us,scenario_032,federal_income_tax_before_refundable_credits,3,llm_error,thresholds_rates,False,,"The decisive rule is the 2026 federal parameter regime: the household receives a $2,200 CTC, split into $500 nonrefundable and $1,700 refundable. The $500 nonrefundable portion fully eliminates pre-credit income tax, while the refundable CTC and $4,427 EITC are excluded from federal_income_tax_before_refundable_credits." -us,scenario_032,federal_refundable_credits,37,llm_error,thresholds_rates,False,,"For 2026, the household gets the full one-child EITC maximum of $4,427. Earned income is $29,000, and AGI is $30,916 after adding $1,916 of taxable Social Security (50% of provisional income above $32,000). AGI stays below the married-filing-jointly phaseout start of about $31,160. Under OBBBA, the CTC is $2,200 per child with a $1,700 refundable cap, and federal tax liability is $0 because the $32,200 standard deduction exceeds AGI, so the ACTC is the full $1,700. Most wrong answers made one of three mistakes: applying the expired pre-TCJA $1,000 ACTC, using the single-filer EITC phaseout threshold or a stale one instead of the MFJ threshold, or ignoring or misstating the $1,700 refundable cap." -us,scenario_032,free_school_meals_eligible,24,llm_error,state_local_rule,False,,"The household's income ($42,664, 1.56 times the poverty guideline) does exceed the federal NSLP 130% FPL free-meal limit, and it has no SNAP/TANF categorical eligibility. Minnesota, however, runs a universal free school meals program (in place since the 2023-24 school year), so PolicyEngine sets school_meal_tier to FREE for every K-12 child in the state regardless of income. Every model that answered stopped at the federal income test or the categorical-eligibility check and never applied Minnesota's universal free-meals override." +us,scenario_032,federal_refundable_credits,38,llm_error,thresholds_rates,False,,"The reference has two parts. The first is the full 2026 one-child EITC maximum of $4,427. AGI is $30,916 ($29,000 wages plus $1,916 taxable Social Security, which is 50% of provisional income above the $32,000 joint base). That AGI and the $29,000 of earnings both fall below the 2026 married-filing-jointly phaseout start, so no phaseout applies. The second part is the refundable CTC at the OBBBA $1,700 per-child cap. Pre-credit tax is zero because AGI is below the $32,200 joint standard deduction, so no CTC is absorbed. Models missed mainly in three ways: they used the pre-TCJA/sunset $1,000 ACTC (15% over $3,000), they applied a single-filer or wrong EITC phaseout threshold, or they misprojected the EITC maximum or refundable CTC cap." +us,scenario_032,free_school_meals_eligible,25,llm_error,state_local_rule,False,,"Minnesota has run a universal free school meals program since the 2023-24 school year, so every K-12 student there gets free breakfast and lunch whatever the family's income. PolicyEngine flags this with state_has_universal_free_school_meals = True and sets school_meal_tier to FREE for the household's one K-12 child. The household's income is 1.56 times the poverty guideline and it has no SNAP/TANF categorical eligibility, so neither fact matters. Every model that answered applied only the federal NSLP test (income at or below 130% FPL, or categorical eligibility) and never checked Minnesota's universal program." us,scenario_032,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_032,head_medicaid_eligible,15,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI (42 CFR 435.603(e), which follows IRC 36B(d)(2)(B)) adds back the non-taxable part of Social Security benefits. The Head's $13,664 in Social Security dependent benefits therefore counts in full alongside the $29,000 in wages. That puts household MAGI at $42,664, or 1.56 times the FPL for a 3-person household, above Minnesota's 138% expansion-adult limit. Minnesota's Medical Assistance limit for parents and caretakers is the same 133% FPL plus the 5% disregard, so the 50-year-old Head qualifies through no pathway. Most models counted only wages, or wages plus the taxable part of Social Security (about $30,916). The rest correctly found $42,664 above 138% but then invented a higher Medicaid limit for parents, or a larger household." +us,scenario_032,head_medicaid_eligible,16,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI is AGI plus the non-taxable portion of Social Security benefits (42 CFR 435.603(e)). The head's full $13,664 of Social Security dependent benefits therefore counts along with the $29,000 in wages, for a MAGI of $42,664. That is about 156% of the 3-person FPL ($27,320), which is above Minnesota's 138% FPL Medical Assistance limit. Minnesota's limit for parents and caretakers is the same 133%+5% standard as for expansion adults, and the head is not aged, disabled, pregnant, or receiving SSI, so no category applies. Most models counted only wages, or wages plus the taxable portion of Social Security (about $30,916, roughly 113% FPL). Two models counted the full $42,664 but then invented a higher limit for parents, and one inflated the household size." us,scenario_032,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_032,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific and requires the applicant to be pregnant, postpartum, breastfeeding, an infant, or a child under age five. A 50-year-old head with all relevant unlisted statuses set to false fails categorical eligibility regardless of household income or another household member's circumstances." -us,scenario_032,payroll_tax,41,prompt_ambiguity,state_local_rule,False,,"The reference adds Minnesota Paid Leave, which began January 1, 2026, to employee FICA ($1,798 Social Security plus $420.50 Medicare). The engine assumes the employer withholds the largest employee share it is allowed to: 0.44%, half of the 0.88% premium, on $29,000, which is $127.60. Minnesota lets the employer pay that share instead, and the prompt says nothing about this. So ""mandatory employee state payroll tax"" can mean (a) the employee's 0.44% share withheld, giving $2,346.10, or (b) nothing, because the employer is allowed to cover the whole premium, giving $2,218.50. Almost every model chose reading (b). The real errors are the few answers that are neither $2,218.50 nor $2,346.10: a wrong Paid Leave rate, invented rates or adjustments, an employer-share mix-up, and missing outputs. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)." -us,scenario_032,reduced_price_school_meals_eligible,26,llm_error,state_local_rule,False,,"Minnesota has universal free school meals, so PolicyEngine gives every student the FREE tier whatever the family's income. Reduced-price eligibility is false only because the better free tier replaces it. Income is not the reason: the household sits at 1.56 of the poverty guideline, inside the 130-185% reduced-price band. Every model that answered placed income in the federal 130-185% band and stopped there. None applied Minnesota's universal free-meals rule, which moves the student out of the reduced-price tier." +us,scenario_032,payroll_tax,42,prompt_ambiguity,state_local_rule,False,,"Most listed models reported employee FICA alone ($1,798 Social Security + $420.50 Medicare = $2,218.50), sometimes rounded to $2,219; others included a Paid Leave share, submitted inconsistent amounts or returned no value. The $127.60 difference between unrounded FICA and the frozen reference is the maximum Minnesota Paid Leave deduction; premiums began January 1, 2026. The total premium is 0.88%, and employers may deduct up to half of it (0.44%) from wages. PolicyEngine assumes the employer withholds that full employee share, so it adds 0.44% × $29,000 = $127.60 as employee state payroll tax. The prompt asks for ""mandatory employee state payroll taxes"" and allows two readings: (1) the withheld Paid Leave share counts, giving $2,346.10; or (2) it does not count, because Minnesota lets the employer pay the whole premium, giving $2,218.50. The prompt never says which choice this employer makes; most listed models reported federal FICA alone, and only grok-4.7 explicitly cited optional paid-leave withholding. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)." +us,scenario_032,reduced_price_school_meals_eligible,27,llm_error,state_local_rule,False,,"Since the 2023-24 school year, Minnesota has provided universal free school meals to every student regardless of income. PolicyEngine therefore puts this household in the FREE tier, and reduced_price_school_meals_eligible is 0 because the free tier supersedes reduced-price. The household's 1.56 FPG ratio falls in the federal 130–185% reduced-price band, but that band never applies in Minnesota. Every model applied the federal NSLP income bands without the state universal-free-meals override." us,scenario_032,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_032,snap,8,llm_error,categorical_eligibility,False,,"SNAP eligibility ends at the gross-income screen for this three-person Minnesota household: $29,000 of wages plus $13,664 of Social Security dependent benefits produces $42,664 of annual gross income, above the applicable limit. Take-up assumptions and benefit-allotment calculations do not create eligibility after that screen fails, so the annual benefit is $0." us,scenario_032,spouse_chip_eligible,4,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person to be ineligible for Medicaid, not merely under age 19 and below the CHIP income ceiling. The 18-year-old spouse qualifies for Minnesota Medicaid under the OLDER_CHILD category, so that Medicaid eligibility precludes CHIP eligibility without regard to whether household income is below the CHIP threshold." @@ -302,16 +302,16 @@ us,scenario_032,spouse_medicaid_eligible,17,llm_error,age_disability,False,,"The us,scenario_032,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_032,spouse_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC does not cover every low-income woman of childbearing age or every adult living with a child. An adult woman must be pregnant, breastfeeding, or within the applicable postpartum period; the spouse has none of these statuses because all unlisted statuses are false, so she fails categorical eligibility regardless of the income test." us,scenario_032,state_income_tax_before_refundable_credits,7,llm_error,state_local_rule,False,,"Minnesota starts from federal AGI ($29,000 wages plus the $1,916 of Social Security made taxable by the §86 provisional-income formula = $30,916), then applies two Minnesota-specific subtractions the models ignored or undersized: the Minnesota Social Security benefit subtraction, which removes the entire $1,916 for a joint filer at this income (far below the phase-out range), and the Minnesota dependent exemption for Child 1 (roughly $5,300, indexed). That leaves $29,000 of Minnesota income against a 2026 joint standard deduction of roughly $30,700 plus the dependent exemption, so Minnesota taxable income is zero and tax before refundable credits is $0. Every wrong answer pairs a too-small joint standard deduction with a missing Social Security subtraction and a missing dependent exemption, manufacturing a few hundred to a few thousand dollars of phantom taxable income taxed at the 5.35% first-bracket rate." -us,scenario_032,state_refundable_credits,47,llm_error,state_local_rule,False,,"Starting in 2023, Minnesota's EITC-linked Working Family Credit was replaced by the Child and Working Families Credit (CWFC), which is one combined refundable credit. A child under 18 generates the per-child child amount, which is inflation-indexed for 2026. The working-family part is only 4% of earnings, capped at the base amount, with extra amounts only for qualifying children aged 18 to 24. AGI of $30,916 is below the married-filing-jointly phase-out threshold, so the full $2,187.60 applies. The wrong answers fall into four groups: they missed the CWFC entirely ($0), stacked an old-style EITC-percentage WFC worth $1,000–$2,000 on top of the child credit, used the unindexed 2023 amounts ($1,750 + $350 = $2,100), or applied a phase-out that does not apply here." +us,scenario_032,state_refundable_credits,48,llm_error,thresholds_rates,False,,"Since 2023, Minnesota has paid a single refundable Child and Working Families Credit (CWFC) in place of the old EITC-linked Working Family Credit. It combines a per-child amount (base $1,750) with a working-family amount equal to 4% of earnings, capped at a base of $350. Both pieces are inflation-indexed, and together they are reduced by one shared 12% phaseout. This joint filer has AGI of $30,916, which is below the indexed joint-filer phaseout threshold, so the full indexed 2026 amounts apply: $2,100 × about 1.042 = $2,187.60 (about $1,823 for the child and about $365 for the working-family cap). Most models fell into one of three errors. Some stacked the old EITC-percentage WFC (often more than $1,000) on top of the child credit. Some used the unindexed $1,750 + $350 = $2,100. Some missed the CWFC entirely and answered $0." us,scenario_033,federal_income_tax_before_refundable_credits,38,prompt_ambiguity,taxable_income_or_deductions,False,,"AGI is $83,451.23: $79,000 wages + $775 interest + the $243 state tax refund + $16,320 taxable Social Security (85% cap) − $11,400 SE loss − $1,487 HSA. Under post-OBBBA 2026 law, deductions are $47,500: the $32,200 MFJ standard deduction, 2 × $1,650 aged additions, and the $6,000-per-senior deduction for both spouses. That leaves $35,951.23 of taxable income, taxed at 10% up to $24,800 and 12% above it, for $3,818.15. The models mostly went wrong in four ways. Many left out the $12,000 senior deduction. Others assumed TCJA had expired, bringing back personal exemptions and the 15% bracket. Some dropped the $243 refund. Others claimed deductions that don't apply here: auto-loan interest, overtime, or invented itemized deductions. Several also made arithmetic slips or reported numbers that don't match their own working. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." us,scenario_033,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_033,payroll_tax,11,llm_error,other,False,,"This is a simple calculation: employee FICA of 7.65% (6.2% Social Security plus 1.45% Medicare) on combined wages of $79,000 gives $6,043.50. Both spouses earn well under the Social Security wage base, the $250,000 joint Additional Medicare Tax threshold doesn't apply, South Dakota has no employee state payroll tax, and the negative self-employment income doesn't touch employee payroll tax. Almost every model either reached $6,043.50 and then submitted a different number (arbitrary rounding, an invented adjustment, or an unexplained replacement) or multiplied 7.65% × $79,000 wrong. One model gave no answer." us,scenario_033,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Self-employment tax is a nonnegative liability assessed only on positive net earnings from self-employment. The listed negative SSTB amount does not create a negative self-employment tax or refundable tax offset, so the self-employment tax base and liability are both zero." -us,scenario_036,federal_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"The calculation is simple: $59,220 of taxable pension income, minus the 2026 OBBBA single standard deduction of $16,100, leaves $43,120 of taxable income. Under the 2026 single brackets, tax is 10% of the first $12,400 ($1,240) plus 12% of the remaining $30,720 ($3,686.40), for $4,926.40, with no nonrefundable credits. The models went wrong in three main ways. Some assumed the TCJA would expire in 2026 and used an $8,300 standard deduction, a personal exemption and a 15% bracket, even though OBBBA made the TCJA rate and deduction structure permanent. Others used stale 2024 or 2025 deductions and brackets. Several did the arithmetic correctly, got about $4,926, and then submitted a different number anyway. A few added deductions or credits the facts don't support: mortgage interest from a balance alone, the elderly credit for a 56-year-old, or medical costs below the 7.5% AGI floor." +us,scenario_036,federal_income_tax_before_refundable_credits,33,llm_error,thresholds_rates,False,,"The tax is set entirely by the 2026 parameters under the One Big Beautiful Bill Act (OBBBA). The single standard deduction is $16,100 and the 10% bracket ends at $12,400, with 12% above that. That gives taxable income of $59,220 − $16,100 = $43,120 and tax of $1,240 + 12% × $30,720 = $4,926.40, with no nonrefundable credits. Many models assumed the TCJA would sunset and used an $8,300 standard deduction, a personal exemption and a 15% bracket. Others used stale 2024 or 2025 deductions and bracket thresholds. Several worked out $4,926 exactly and then replaced it with a different number for no stated reason." us,scenario_036,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New Jersey Medicaid requires both membership in a qualifying eligibility category and satisfaction of that category's financial rules. This 56-year-old has medicaid_category NONE, and MAGI income of 3.71 times FPL exceeds the limits for all applicable MAGI pathways; the listed expenses and assets do not create another pathway." us,scenario_036,payroll_tax,1,llm_error,payroll_tax_base,False,,"Taxable private pension income is not wages and is excluded from the employee Social Security, Medicare, and mandatory state payroll-tax bases. With no wages or other earned income listed, every employee-side payroll-tax component is zero." us,scenario_036,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_036,state_income_tax_before_refundable_credits,31,llm_error,taxable_income_or_deductions,False,,"The head is 56 and not disabled, so New Jersey's pension exclusion (age 62+ or disabled only) does not apply and the full $59,220 is NJ gross income. The only subtraction is the $1,000 single personal exemption. No medical deduction survives: the $1,000 'health insurance premiums excluding Part B' is the total that already includes the $1,000 'other health insurance premiums', employer-sponsored premiums are not out-of-pocket deductible, and OTC items don't count, which leaves about $1,100 of qualifying expenses, below the 2% floor of $1,184.40. The tax on $58,220 is $280 + $262.50 + $175 + 5.525% × $18,220 = $1,724.16. The models fell into three groups: those that applied the age-62 pension exclusion, those that built a medical deduction by double-counting premiums or adding the $7,389 ESI premiums, and those that used invented brackets, standard deductions, or numbers that didn't follow from their own work." +us,scenario_036,state_income_tax_before_refundable_credits,32,llm_error,taxable_income_or_deductions,False,,"The head is 56 and not disabled, so New Jersey's pension exclusion doesn't apply; it requires age 62 or older, or disability. The full $59,220 pension is NJ gross income. The $1,000 single exemption is the only subtraction, which leaves $58,220 of taxable income. The single brackets give $280 + $262.50 + $175 + 5.525% × $18,220 = $1,724.16. No medical-expense deduction enters NJ taxable income. The ""$1,000 excluding Part B"" line restates the $1,000 of other premiums rather than adding to it. The $7,389 of ESI premiums and the OTC spending are not counted as deductible out-of-pocket costs. That leaves $1,100 of countable expenses, which is below the 2% floor of $1,184.40. Most models went wrong in one of three ways: they applied the age-62 pension exclusion at 56, they built a medical deduction by stacking ESI premiums, duplicate premium lines, or OTC costs, or they invented a standard deduction or rates that NJ doesn't have." us,scenario_036,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_037,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The correct path starts with AGI of $41,539, which is wages minus the $386 traditional 401(k) and the $18 traditional IRA. The ESI premiums are not subtracted again. Taxable income then subtracts both the $16,100 2026 OBBBA standard deduction and the $9,629 qualified overtime deduction. The overtime deduction comes off below the line, so it does not lower AGI. That leaves $15,810 of taxable income, taxed at 10% on the first $12,400 and 12% on the rest, for $1,649.25. Most models missed the overtime deduction or assumed the TCJA sunset. Others subtracted ESI premiums from wages, used 2024/2025 brackets, or claimed a Saver's Credit, even though the $41,539 AGI is above the single filer's eligibility ceiling." us,scenario_037,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A 44-year-old single filer with no dependent children or qualifying education expenses and $41,943 of wages receives neither EITC nor refundable CTC. All other refundable federal credit components evaluated to zero, so federal refundable credits total $0." @@ -327,7 +327,7 @@ us,scenario_038,child2_head_start_eligible,4,llm_error,categorical_eligibility,F us,scenario_038,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Louisiana evaluates the 6-year-old under the older-child Medicaid category using household MAGI relative to the applicable child income limit. The household's MAGI is 0.66 times FPL, so the child satisfies that category's income test; the listed assets do not disqualify a MAGI-category child." us,scenario_038,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday, so a six-year-old is categorically ineligible regardless of household income. All four models incorrectly treated age six as falling within the WIC child category; the models that also evaluated income therefore never reached the correct result because they first misapplied the age rule." us,scenario_038,federal_income_tax_before_refundable_credits,2,llm_error,other,False,,"The requested amount cannot fall below zero: taxable income and income tax before credits are zero, and nonrefundable credits used are capped at the existing tax liability. Refundable credits are excluded from this output, so the resulting federal income tax before refundable credits is $0." -us,scenario_038,federal_refundable_credits,41,llm_error,thresholds_rates,False,,"Earned income is $7,468 in wages plus $15,525 in self-employment income, minus half of the SE tax (about $1,097), which comes to about $21,896. That puts the couple filing jointly with two children on the 2026 EITC plateau: past the phase-in end of about $18,290 and below the joint phase-out start of about $31,000. So the EITC is the full 2026 maximum of $7,316. Under the OBBBA rules for 2026, the CTC is $2,200 per child and the refundable cap is $1,700 per child, so the refundable CTC is the smaller of $3,400 and 15% × (earned income − $2,500), which is $2,909.35. Models mostly missed in three ways: they used the wrong EITC maximum, they applied the pre-TCJA $1,000-per-child CTC with a $3,000 floor, or they mismeasured the ACTC's earned-income base, either by skipping the 15% limit or by calculating net SE earnings wrongly." +us,scenario_038,federal_refundable_credits,42,llm_error,thresholds_rates,False,,"The reference is the 2026 two-child EITC at its maximum of $7,316 plus the refundable CTC of $2,909.35. Earned income is about $21,896, which is in the EITC plateau and below the joint-filer phase-out start. The refundable CTC follows 2026 law: $1,700 refundable per child, so the cap is $3,400, but it is limited to 15% of earned income above $2,500. Earned income here means wages plus self-employment income minus the deductible half of SE tax, so 15% × ($21,896 − $2,500) ≈ $2,909. Most models went wrong in one of three ways: they used a stale or invented EITC maximum, they applied the expired pre-2018 $1,000-per-child ACTC (a $2,000 cap) or the full $3,400 cap without the 15% earnings limit, or they measured earned income incorrectly (gross SE income, 92.35% of SE income, or a double reduction)." us,scenario_038,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The household qualifies for the free tier through two independent routes: SNAP receipt confers categorical eligibility, and school-meal income is 70% of the federal poverty guideline, below the 130% cutoff. The two K–12 children therefore generate positive annual free-school-meal support, so the binary output is Yes." us,scenario_038,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_038,head_medicaid_eligible,4,llm_error,thresholds_rates,False,,"Louisiana has run the ACA adult expansion since 2016, so a 25-year-old non-dependent adult who does not receive SSI falls in the MAGI-based ADULT category, which has a 138% FPL limit. The head's household MAGI is about $22,992 from $7,468 in wages plus $15,525 in self-employment income. For a household of four that is 0.66x FPL, far below 138%, so the head is eligible. All four models said the income was above the limit. None of them compared household MAGI to the poverty guideline for four people, and some confused the expansion limit with Louisiana's much lower pre-expansion parent limit." @@ -335,19 +335,19 @@ us,scenario_038,head_wic_eligible,3,llm_error,categorical_eligibility,False,,"WI us,scenario_038,payroll_tax,5,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes only employee-side FICA on the head’s wage earnings; self-employment income belongs in the separate self-employment-tax output. PolicyEngine applies the underlying wage amount and component-level calculations to produce $462.98 of Social Security tax plus $108.28 of Medicare tax, totaling $571.26." us,scenario_038,reduced_price_school_meals_eligible,2,llm_error,thresholds_rates,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a general test of whether income is below 185% of the federal poverty guideline. At 70% of the guideline, and with categorical eligibility, this household receives the superior free-meals tier, so reduced-price support is zero." us,scenario_038,self_employment_tax,9,llm_error,payroll_tax_base,False,,"The calculation is a single step: net earnings from self-employment = $15,525 × 0.9235 = $14,337.34. That amount plus the $7,468 in wages is far below the Social Security wage base, so the full 15.3% rate applies and SE tax = $14,337.34 × 0.153 = $2,193.61. The deduction for half of SE tax lowers AGI only. It does not lower the SE tax base again, because the 92.35% factor already accounts for it. Most models used the right method but made arithmetic or transcription slips. One wrongly applied the half-SE-tax deduction a second time, and one gave no answer." -us,scenario_038,snap,39,llm_error,thresholds_rates,False,,"The calculation is simple. Monthly earned income is $1,916.04. Subtract the 20% earned income deduction ($383.21) and the FY2026 four-person standard deduction of $223, which leaves net income of $1,310. 30% of that is $393, and subtracting it from the FY2026 maximum allotment of $994 gives $601 a month, or $7,212 a year. There are no shelter, dependent care or medical deductions, because no rent or mortgage payment is listed and no member is elderly or disabled. Most wrong answers came from using stale or guessed parameters, such as the FY2025 values of $975 and $204, a $1,000–$1,050 allotment, or a $198–$217 standard deduction. Others invented deductions, made arithmetic slips, or wrongly found the household ineligible." +us,scenario_038,snap,40,llm_error,thresholds_rates,False,,"The setup is simple. The household has $1,916.04 a month of earned income and no shelter, dependent-care or countable medical costs. The 20% earned income deduction ($383.21) and the FY2026 four-person standard deduction of $223 leave net income of $1,310. The FY2026 four-person maximum allotment is $994, and $994 − 30% × $1,310 ($393) = $601 a month, or $7,212 a year. Most models used the right formula but wrong parameter values: older standard deductions ($198–$217), maximum allotments other than $994 ($879–$1,050), or no whole-dollar rounding. Others made up shelter or dependent-care deductions, or wrongly found the household ineligible even though its income is far below both income tests and it is categorically eligible through BBCE." us,scenario_038,spouse_chip_eligible,4,llm_error,health_coverage,False,,"CHIP eligibility requires the person to be ineligible for Medicaid, not merely under age 19 and below the CHIP income ceiling. The spouse qualifies for Medicaid under Louisiana's OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility without regard to whether other coverage was listed." us,scenario_038,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The trap is that the 18-year-old is labeled \""spouse,\"" and all three models let that label override the age-based pathway: PolicyEngine assigns her the OLDER_CHILD medicaid_category, which covers ages 6-18 in Louisiana and does not require being a tax dependent or an unmarried child. The income test is not close either — household MAGI is 0.66 x FPL, below Louisiana's ~142% FPL older-child limit and below the 138% FPL adult expansion limit, so both the child pathway and the adult pathway the models invoked return eligible. Every wrong answer therefore combines a category error (adult instead of older child) with an unperformed or inverted FPL comparison, and one model additionally imported a nonexistent asset test into a MAGI category." us,scenario_038,spouse_wic_eligible,5,llm_error,categorical_eligibility,False,,"WIC income eligibility does not independently confer eligibility: a person must also be pregnant, breastfeeding, postpartum, an infant, or a child under age five. The spouse is 18, all unlisted statuses are false, and the only children are ages six and seven, so neither parenthood, childbearing age, nor low household income places the spouse in a WIC categorical group." us,scenario_038,state_income_tax_before_refundable_credits,10,llm_error,state_local_rule,False,,"Louisiana's 2024 tax reform took effect in 2025. It replaced the graduated brackets and the old combined personal exemption–standard deduction ($9,000 MFJ, plus $1,000 per dependent) with a flat 3% rate and a $25,000 standard deduction for joint filers. Federal AGI is $21,896: $22,993 of wages and SE income minus the half-SE-tax deduction of about $1,097. That is below the $25,000 MFJ deduction, so Louisiana taxable income is zero and so is the tax. Most models used pre-reform exemptions and brackets (1.85%, 2%/4%) or applied only the $12,500 single-filer deduction to a joint return, which left positive taxable income. One model reached $0 in its reasoning but submitted a different number." -us,scenario_038,state_refundable_credits,37,llm_error,state_local_rule,False,,"Louisiana's EITC equals 5% of the federal EITC and is refundable. In this case it is the only state refundable credit. The household files jointly with two qualifying children and has earned income of about $21,900 (wages of $7,468 plus net self-employment earnings after the SE-tax deduction). That puts it on the federal EITC plateau, well below the MFJ phase-out start of about $31k, so the federal credit is the 2026 two-child maximum of $7,316, and 5% of that is $365.80. Most wrong models said Louisiana has no refundable EITC, or that its EITC is nonrefundable. The rest used the right structure but the wrong federal maximum, the wrong state percentage (3.5% or 50%), or wrongly applied the phase-out." +us,scenario_038,state_refundable_credits,38,llm_error,state_local_rule,False,,"Louisiana's EITC equals 5% of the federal EITC and is refundable, so it pays out even when the household owes no Louisiana tax. Earned income is about $21,896: $7,468 in wages plus $15,525 of self-employment income, minus about $1,097 (half the SE tax). That puts this married-filing-jointly household with two children on the 2026 federal EITC plateau, so the federal credit is the maximum of $7,316. Five percent of that is $365.80. About half the models said Louisiana has no refundable credit, or that its EITC is nonrefundable or repealed, and answered $0. Most of the rest used the right 5% rate but got the federal EITC maximum wrong, or used a 3.5% or 50% rate." us,scenario_039,federal_income_tax_before_refundable_credits,43,llm_error,household_unit_or_filing_status,False,,"The case turns on filing status. Qualifying-surviving-spouse status under IRC §2(a) requires keeping a home for a dependent child. No dependent is listed, so the 'surviving spouse' flag does not unlock joint rates, and the head files single. That gives 85% of Social Security taxable under the single thresholds ($30,689.25), AGI of $79,209.22, the 2026 OBBBA single standard deduction of $16,100, and taxable income of $63,109.22. Single brackets (10% to $12,400, 12% to $50,400, 22% above) then produce $8,596.03. Most wrong answers used qualifying-surviving-spouse joint brackets and the $32,200 joint deduction. A second group assumed the TCJA sunset (which OBBBA repealed) or used stale 2025 deductions and brackets. A few invented credits or extra deductions that zeroed out the tax." us,scenario_039,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_039,head_medicaid_eligible,3,llm_error,health_coverage,False,,"Every Virginia Medicaid pathway open to a 61-year-old is income-tested: the ACA expansion adult group caps MAGI at 138% FPL, and the SSI-linked aged/blind/disabled group sits far below that and keys off SSI receipt, which is $0 here. This head's Medicaid MAGI counts the full $36,105 Social Security benefit (untaxed portion included) plus $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, offset only by a $6,260 self-employment loss, landing at 368% FPL — 2.7x the expansion limit — so the engine assigns medicaid_category NONE. All three models either asserted without arithmetic that MAGI fell under 138% FPL or treated the ""is disabled"" flag as a categorical pass that skips the income test entirely. Disability and the expansion group's waived asset test change which income test applies, never whether one applies." us,scenario_039,head_medicare_eligible,20,llm_error,age_disability,False,,"Before age 65, Medicare eligibility through disability requires 24 months of receiving Social Security Disability Insurance. Being flagged as disabled is not enough. PolicyEngine checks age ≥ 65, at least 24 months of SSDI receipt, or end-stage renal disease (ESRD). The head is 61, has no listed SSDI months (unlisted inputs count as 0), and has no ESRD. Their $36,105 of Social Security is retirement income, not disability benefits. Every model treated the disabled flag by itself as enough for Medicare, and that is why they answered 1 instead of 0." us,scenario_039,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary income, not estate income, Social Security retirement benefits, IRA distributions, private pensions, or self-employment income. No wages were listed, and the prompt requires unlisted numeric inputs to be zero, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax bases are all zero." us,scenario_039,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_039,state_income_tax_before_refundable_credits,44,llm_error,thresholds_rates,False,,"Virginia AGI is $48,519.97: federal AGI of $79,209.22 minus the $30,689.25 of federally taxable Social Security. Because the head has no dependent, a surviving spouse files as single and gets the 2026 single standard deduction of $8,750 plus one $930 exemption. That leaves taxable income of $38,839.97, and the tax is $720 + 5.75% × $21,839.97 = $1,975.80. Most models made the Social Security subtraction correctly but missed on the deduction. Some used the old $8,500 or $8,000 amount, which gives $1,990 or $2,019. Others gave the surviving spouse the joint $17,000–$17,500 deduction, which gives about $1,473. A smaller group skipped the Social Security subtraction or invented age, disability or retirement exclusions that don't apply to a 61-year-old with no disability income." +us,scenario_039,state_income_tax_before_refundable_credits,45,llm_error,thresholds_rates,False,,"Virginia starts from federal AGI of $79,209.22 and subtracts all federally taxable Social Security ($30,689.25), leaving VA AGI of $48,519.97. The head is 61 with no qualifying dependent, so she files as single (Virginia has no surviving-spouse status) and gets no age deduction or disability subtraction. The correct deductions are the 2026 single standard deduction of $8,750 and the $930 personal exemption, giving taxable income of $38,839.97 and tax of $720 + 5.75% × $21,839.97 = $1,975.80. The largest group of wrong answers used the older $8,500 deduction and landed on $1,990.18. Other groups used the $17,500 joint deduction for a ""surviving spouse"" ($1,472.68), skipped the Social Security subtraction (about $3,740), or invented age or disability exclusions (about $0)." us,scenario_039,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_040,federal_income_tax_before_refundable_credits,9,llm_error,taxable_income_or_deductions,False,,"Every model got the same AGI the engine used ($28,000 wages + $4,300 taxable Social Security = $32,300, since workers' compensation is excluded and provisional income of $40,600 falls in the 50% tier between the $32,000 and $44,000 MFJ thresholds) or missed it, and then lost the case on the deduction side. For tax year 2026 the TCJA structure is permanent: personal exemptions remain repealed, the MFJ standard deduction is $32,200, the aged additional standard deduction is $1,650 per 65+ spouse ($3,300), and the OBBBA senior deduction adds $6,000 per spouse aged 65+ ($12,000) for a total of $47,500 — well above the $32,300 AGI, so taxable income and tax are $0. Six of the nine models instead applied a TCJA-sunset 2026 with a ~$16,500–$20,000 standard deduction plus revived personal exemptions of ~$10,200–$10,600, a combination that leaves a small positive taxable income taxed at 10%. The two that used a post-OBBBA standard deduction still failed by inflating includible Social Security or by treating the senior deduction as a tax offset rather than a $6,000-per-spouse deduction." us,scenario_040,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -359,11 +359,11 @@ us,scenario_040,spouse_medicaid_eligible,4,llm_error,categorical_eligibility,Fal us,scenario_040,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility applies at age 65, so the 69-year-old spouse qualifies through the age pathway without needing disability-based eligibility or 24 months of SSDI receipt. The spouse’s workers’ compensation and disability facts do not alter that age-based result." us,scenario_040,state_income_tax_before_refundable_credits,6,llm_error,taxable_income_or_deductions,False,,"The case turns on two Arizona-specific rules: Arizona subtracts from federal AGI all Social Security benefits included there, and Arizona's standard deduction conforms to the base federal married-filing-jointly amount (about $32,200 for 2026) with no additional age-65 amount. Removing the $4,300 of taxable Social Security leaves $28,000 of wages as Arizona AGI, which is more than $4,000 below the standard deduction, so taxable income is zero and the 2.5% flat tax yields nothing before refundable credits; itemizing is worse at $16,324 of mortgage interest plus real estate taxes. The wrong answers all manufacture a positive base either by shrinking the deduction — inventing $20,000, $22,000, or an unstated ~$22,700 figure, in two cases by grafting a federal-style age-65 add-on Arizona does not have — or by leaving Social Security in the Arizona base. One model produced the zero result and then overrode it, and one returned nothing at all." us,scenario_040,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"Arizona’s refundable credits are not awarded merely because a married filer is elderly and has income below a general threshold. The household satisfies no applicable Arizona refundable-credit eligibility pathway in 2026, so az_refundable_credits and the aggregate state_refundable_credits are both zero; the other failure is a missing required output." -us,scenario_042,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The head is a widowed person with no dependent child, so they file as Single, not Qualifying Surviving Spouse. Taxable Social Security uses the single $25k/$34k thresholds: provisional income of $48,525 gives $16,846.25 taxable and AGI of $50,580.71. Under 2026 OBBBA law (TCJA made permanent), the deductions are the $16,100 standard deduction, the $2,050 aged add-on and the $6,000 senior deduction, for $24,150 total. Taxable income is $26,430.71, of which $4,681 of qualified dividends and capital gains is taxed at 0% and $21,749.71 is taxed at 10%/12%, giving $2,361.96. Most wrong answers came from one of three mistakes: using QSS joint parameters, leaving out the $6,000 senior deduction, or applying the pre-TCJA sunset schedule with personal exemptions. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)." +us,scenario_042,federal_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"The head is widowed but has no dependent child, so they file as single. That means single Social Security thresholds of $25k/$34k, which make $16,846 of benefits taxable. AGI is $50,580.71. Deductions are the $18,150 standard deduction for an aged single filer plus the $6,000 OBBBA senior deduction, for a total of $24,150 and taxable income of $26,430.71. The $4,681 of qualified dividends and gains is taxed at 0%, and the $21,749.71 of ordinary income is taxed at 10% up to $12,400 and 12% above, giving $2,361.96. Most of the wrong answers came from three mistakes: using qualifying-surviving-spouse joint parameters, leaving out the $6,000 senior deduction, or applying the expired pre-TCJA schedule with personal exemptions and a 15% bracket. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)." us,scenario_042,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_042,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax uses employee wages as its base, and the prompt lists no wage or salary income. The pension, IRA distributions, investment income, Social Security benefits, and farm loss do not generate employee-side Social Security or Medicare tax, so employee payroll tax is $0." us,scenario_042,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_042,state_income_tax_before_refundable_credits,47,reference_engine_defect,state_local_rule,False,"The reference federal AGI of $43,637.65 is exactly $29,981.71 of non-Social Security income plus $13,655.95 of taxable Social Security, where 0.85×($29,981.71+$14,790−$34,000)+$4,500 = $13,655.95; that sum can only come out this way if PolicyEngine left the $3,753 of non-Schedule-D capital gain distributions out of gross income, which conflicts with IRC §61(a)(3) and Form 1040 line 7. Separately, the Wisconsin subtractions leave out the 2025 Wisconsin Act 15 retirement income exclusion for filers age 67 and older (up to $24,000), which would cover almost all of this filer's $24,086 of IRA and pension income.","The reference gets Wisconsin AGI of $29,981.71 by subtracting only the $13,655.95 of taxable Social Security from a federal AGI that contains none of the $3,753 in capital gains, and it applies no retirement-income subtraction and no 30% capital-gain subtraction. It then takes the single sliding-scale standard deduction ($12,671.36) and $950 of exemptions, taxes $16,360.34 at 3.5%/4.4% to get $584.74, and subtracts the $300 school property tax credit (12% of the first $2,500 of property tax). The wrong answers fall into three groups. Models that applied the $24,000 age-67+ retirement exclusion got $0. Models that kept the $3,753 of gains (less the 30% long-term exclusion) and otherwise followed the same steps got about $415–$430. The rest misapplied the standard deduction, filing status, exemptions, rates, or the $300 credit (skipping it, overstating it, or inventing other credits). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)." +us,scenario_042,state_income_tax_before_refundable_credits,31,llm_error,taxable_income_or_deductions,False,,"The 79-year-old filer qualifies for Wisconsin's retirement income exclusion for filers 67 and older (Schedule SB line 16). It has no income limit and removes up to $24,000 of pension and IRA income for a single or surviving-spouse filer. Here the full $24,000 cap applies against $24,086 of retirement income. That drops Wisconsin AGI from $32,608.81 to about $8,609, which the standard deduction and $950 of exemptions fully cover, so tax is $0. The filer elects that path over the standard path's $408.70. Nearly every model stopped at the standard path: subtract Social Security, sometimes the 30% capital-gain exclusion, apply the sliding standard deduction and exemptions, then sometimes the $300 school property tax credit. Their answers cluster around $400–$1,000 because none applied the $24,000 exclusion." us,scenario_042,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"The Wisconsin homestead credit requires qualifying household income and property-tax circumstances to be computed from the stated annual inputs under the 2026 rules. The household receives substantial Social Security, IRA, pension, dividend, interest, and capital-gain income, so reducing household income to roughly $14,420 and applying a maximum-credit shortcut creates a refundable credit that the full eligibility computation does not allow." us,scenario_043,federal_refundable_credits,4,llm_error,age_disability,False,,"The single trap is the upper age limit on the childless EITC: IRC §32(c)(1)(A)(ii)(II) allows the no-qualifying-children credit only to filers at least 25 and under 65 at the close of the tax year, and the American Rescue Plan's one-year suspension of that ceiling applied to 2021 only. The head is 66 in tax year 2026, so the EITC is $0 regardless of the $4,088 in earnings sitting squarely in the phase-in range, and with no children, no education expenses, and no Recovery Rebate or refundable payroll tax credit, total federal refundable credits are $0. All four models jumped straight to the phase-in arithmetic on earned income and never applied the age test, three of them landing on 7.65% × ~$4,088 ≈ $312–313 and one on a fabricated 15.3% rate." us,scenario_043,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -393,25 +393,25 @@ us,scenario_045,payroll_tax,10,llm_error,payroll_tax_base,False,,"The trap is th us,scenario_045,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_045,snap,4,llm_error,categorical_eligibility,False,,"This one-person household has no elderly or disabled member, so it must either pass the 130% FPL gross income test or qualify through Michigan's broad-based categorical eligibility (BBCE), which has a 200% FPL gross limit. Michigan counts child support paid in gross income and deducts it only when computing net income (7 CFR 273.9(d)(5)). That leaves gross income at the full $3,022.97 a month, about 232% of the guideline, which fails both limits, so SNAP is $0 for the year. The wrong answers came from three errors: taking child support or health premiums out of gross income, still paying a benefit after finding the household over the limit, and a submitted value that contradicts the model's own finding that the household is ineligible." us,scenario_045,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_045,state_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"Michigan's tax is a flat 4.25% applied to federal AGI ($36,275.59, the full wage amount; the stated employer-sponsored insurance premiums are not subtracted again) minus the 2026 personal exemption of $5,900. That gives $30,375.59 × 0.0425 = $1,290.96, with no nonrefundable credits. Most wrong answers came from projecting the exemption from memory ($5,000, $5,200, $5,600, $5,800, $6,000 or $6,200) instead of using $5,900. A second group subtracted the $21,208 ESI premiums from AGI and landed near $390–$402. A third group invented a standard deduction or treated the refundable Homestead Property Tax Credit as nonrefundable and reported $0 or close to it. The rest computed a figure near the right one and then submitted an unrelated number." -us,scenario_045,state_refundable_credits,43,llm_error,state_local_rule,False,,"The whole $760.79 is Michigan's refundable Homestead Property Tax Credit for a renter, with no state EITC. The correct steps are: count 23% of the $10,560 rent as property tax ($2,428.80). Subtract 3.2% of total household resources, which equal the full $36,275.59 in wages, giving $1,160.82 and a remainder of $1,267.98. Then apply the 60% non-senior rate to get $760.79. Most wrong answers left out the renter credit entirely and only looked at the zero Michigan EITC. The rest changed one input of the formula: they used 20% (or 17%) of rent instead of 23%, a 3.5% threshold instead of 3.2%, or a different credit rate. Others subtracted the $21,208 employer insurance premiums or the $5,200 child support from household resources, or added a federal EITC that a childless filer at $36k cannot get." +us,scenario_045,state_income_tax_before_refundable_credits,41,llm_error,thresholds_rates,False,,"Michigan tax here is 4.25% × (federal AGI of $36,275.59 − the 2026 personal exemption of $5,900) = $1,290.96. Michigan has no standard deduction and no nonrefundable credit that applies to this filer. The largest group of models used an outdated or guessed exemption ($5,000, $5,200, $5,600, $5,800 or $6,000) or the 2023-only 4.05% rate. A second group subtracted the $21,208 employer insurance premiums from the stated gross wages, which brought AGI down to $15,068. Others got to $0 by inventing a standard deduction, deducting child support, or treating the refundable Homestead Property Tax Credit as nonrefundable. A few computed a number close to the reference and then submitted a different value." +us,scenario_045,state_refundable_credits,44,llm_error,state_local_rule,False,,"The whole $760.79 is Michigan's refundable Homestead Property Tax Credit, which renters can claim. The calculation is 23% of the $10,560 rent ($2,428.80), minus 3.2% of total household resources ($36,275.59, i.e., full wages with no deduction for child support paid or ESI premiums), which leaves $1,267.98, times 60% = $760.79. Federal EITC is $0 for this childless filer at $36k, so the Michigan EITC adds nothing. The wrong answers fall into four groups: models that ignored the homestead credit or thought renters can't claim it; models that used the wrong rent share (20% or 17%), the wrong threshold (3.5%) or the wrong credit rate; models that shrank household resources by subtracting the $21,208 ESI premiums or the $5,200 child support; and models that invented a positive EITC." us,scenario_046,child1_chip_eligible,1,llm_error,state_local_rule,False,,"Oklahoma covers school-age children through Medicaid (SoonerCare), not through a separate CHIP. The engine therefore gives a 10-year-old in Oklahoma no CHIP pathway on age grounds. The household's MAGI of about $92,003 (wages plus interest) is also far above any Oklahoma children's income limit for a family of four. A wrong Yes comes from applying a generic, high CHIP income limit and understating MAGI, instead of Oklahoma's own CHIP rules for this child's age and income." us,scenario_046,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A dependent child qualifies for Oklahoma Medicaid only through an applicable eligibility category. At age 10 and MAGI income of 2.79 times FPL, child1 exceeds the income limits for the regular children's group, CHIP, and every other applicable pathway, so the Medicaid category is NONE and eligibility is No." us,scenario_046,child2_chip_eligible,1,llm_error,taxable_income_or_deductions,False,,"Child2's CHIP eligibility depends on the family's Medicaid/CHIP MAGI measured against the federal poverty line for a family of four. That MAGI is the full $92,003 of wages plus interest, which is about 280% FPL. The $14,839 FLSA overtime premium is already inside the $92,000 of wages. The 2025-2028 overtime deduction is taken below the line, in computing taxable income, so it never lowers AGI or MAGI. Once the overtime premium is subtracted from MAGI, income falls into the CHIP band. Kept in, income is above Oklahoma's CHIP ceiling for a 9-year-old, and child2 is ineligible because Medicaid is also unavailable." us,scenario_046,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Oklahoma Medicaid eligibility requires the child to qualify through an available categorical pathway and its applicable income limit. Child 2 has no qualifying Medicaid category, receives no SSI, and the household's MAGI income of 2.79 times FPL exceeds every applicable child Medicaid threshold, so eligibility is No." -us,scenario_046,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"Under OBBBA rules for 2026, the household's $92,003 AGI is reduced by the $32,200 MFJ standard deduction and by the qualified overtime deduction of $14,838.71. That deduction is the FLSA overtime premium under the $25,000 joint cap. The result is $44,964.29 of taxable income. Tax is $2,480 at 10% up to $24,800 plus 12% on the remainder, or $4,899.71. The full $4,400 CTC ($2,200 per child) is used as a nonrefundable credit because it is below that liability, which leaves $499.71. Itemizing does not win: the $12,000 premium is one amount, reported under two labels; OTC costs do not count; and medical above 7.5% of AGI, mortgage interest and SALT total about $28.7k. AOTC is $0 because no qualified expenses are listed. Most wrong answers omitted the overtime deduction, applied pre-TCJA or 2025 parameters, double-counted premiums to justify itemizing, invented AOTC, or mishandled the nonrefundable CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"Under OBBBA rules for 2026, the household takes the $32,200 MFJ standard deduction plus the below-the-line qualified overtime deduction of about $14,839. The overtime deduction is under the $25,000 joint cap and AGI is far below the phase-out. That gives taxable income of $44,964.29 and tax of $2,480 + 12% × $20,164.29 = $4,899.71. The full $4,400 CTC ($2,200 × 2) is used as a nonrefundable credit, which leaves $499.71. Most wrong answers left out the overtime deduction, used pre-TCJA law (personal exemptions, $1,000 CTC), itemized after double-counting the duplicated $12,000 premium field and non-deductible OTC costs, claimed an AOTC with no qualified expenses, or did not subtract the CTC." us,scenario_046,federal_refundable_credits,14,llm_error,categorical_eligibility,False,,"None of the refundable credits pays out, so federal refundable credits are $0. The EITC is fully phased out at $92,003 AGI for a two-child joint return. The American Opportunity Credit needs qualified tuition and related expenses, and none are listed, so both its nonrefundable and refundable parts are $0. The $4,400 CTC (2 x $2,200) is fully used against tax owed before credits. Even after the $32,200 standard deduction and the $14,839 overtime deduction, that tax is about $4,900, and the refundable CTC only pays back CTC left over after tax, so it is $0. The wrong answers fall into three groups: some made up tuition to claim a $1,000 refundable AOTC, some paid the $1,700-per-child refundable cap without first applying the CTC to tax, and some put tax below $4,400 so a small slice of CTC looked refundable." us,scenario_046,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_046,payroll_tax,17,llm_error,payroll_tax_base,False,,"The head's $92,000 of gross wages is the FICA base. Employee Social Security is 6.2% × $92,000 = $5,704 and employee Medicare is 1.45% × $92,000 = $1,334, for a total of $7,038. There is no Additional Medicare Tax and no Oklahoma employee payroll tax. The most common mistake was subtracting the $9,208 employer-sponsored insurance premium as if it were a Section 125 pre-tax reduction, which the household facts never state, giving $6,334. The other wrong answers came from counting the overtime premium twice, inventing a state payroll tax, arithmetic or transcription slips, or giving up with $0." us,scenario_046,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_046,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Oklahoma taxable income is $75,303: federal AGI of $92,003, less the fixed $12,700 joint standard deduction and four $1,000 exemptions. Oklahoma allows itemizing only when the filer itemizes federally, and the 2026 federal standard deduction of $32,200 is larger than this household's itemized deductions. The 2026 joint schedule under HB 2764 taxes the first $7,500 at 0%, $7,500–$9,800 at 2.5%, $9,800–$14,400 at 3.5%, and income above that at 4.5%, which gives $2,959.14. The nonrefundable Oklahoma child credit is 5% of the $4,400 federal CTC ($2,200 per child under OBBBA), or $220, leaving $2,739.14. Most models used the retired 0.25%–4.75% schedule, itemized or took the federal standard deduction on the Oklahoma return, cut AGI by the federal overtime deduction or ESI premiums, or left out or understated the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"Oklahoma starts from federal AGI of $92,003. The household does not itemize federally because the $32,200 federal standard deduction is larger than its itemized deductions, so Oklahoma requires its own fixed $12,700 MFJ standard deduction, even though its state itemized total would be higher. Subtracting that deduction and four $1,000 exemptions leaves $75,303 of taxable income. The 2026 restructured MFJ schedule is 0% up to $7,500, 2.5% to $9,800, 3.5% to $14,400 and 4.5% above that. It gives $57.50 + $161 + $2,740.64 = $2,959.14, and the nonrefundable credit of 5% of the $4,400 federal CTC ($220) brings tax to $2,739.14. Most models used the old 0.25%–4.75% brackets, itemized on the Oklahoma return, borrowed the federal standard deduction or overtime deduction, or dropped or undercounted the $220 credit." us,scenario_046,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_048,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Age 85 establishes membership in an aged population but does not itself confer Medicaid eligibility. The head receives no SSI, is not a dependent, and qualifies through none of the Medicaid pathways; the 2.17-times-FPL MAGI level also does not establish eligibility in a MAGI category. Low assets cannot substitute for satisfying a pathway's income and categorical requirements." us,scenario_048,head_medicare_eligible,1,llm_error,other,False,,"Medicare eligibility is satisfied because the head is age 85, well above the age-65 threshold. The model’s prose reached that result but its submitted binary value reversed it: eligible must be encoded as 1, not 0." us,scenario_048,payroll_tax,2,llm_error,other,False,,"Payroll tax here is just employee FICA on $7,656 of wages: 6.2% Social Security ($474.67) plus 1.45% Medicare ($111.01), for $585.68. Ohio has no mandatory employee state payroll tax. Social Security survivor benefits are not covered wages. Both models arrived at about $586 in their reasoning but then put a different, unsupported number in the numeric value field. The failure is in transcribing the answer, not in the tax rule." us,scenario_048,snap,5,llm_error,thresholds_rates,False,,"The household’s $34,626 annual income exceeds the applicable $17,420 SNAP eligibility threshold for a one-person Ohio household in 2026, so SNAP eligibility ends before any minimum-allotment calculation. A minimum benefit applies only to an otherwise eligible household; it cannot override a failed income test." us,scenario_048,state_income_tax_before_refundable_credits,2,llm_error,thresholds_rates,False,,"The trap is Ohio's zero-tax bracket. Ohio's individual income tax schedule imposes no tax on Ohio taxable nonbusiness income at or below roughly $26,050, so this filer's $7,656 in wages — the only Ohio-taxable income, since the $26,970 in Social Security survivor benefits stays out of federal AGI at a provisional income of $21,141, below the $25,000 base amount — produces $0 of liability before any credit. Both models correctly excluded the Social Security benefits and then applied the 2.75% rate starting at the first dollar of taxable income, converting a below-threshold filer into a taxpayer. Their differing answers ($106 vs. $94.54) come only from downstream inventions — a fictitious Ohio standard deduction in one case, a recharacterized exemption credit plus the $50 senior citizen credit in the other — layered on the same missing zero bracket." -us,scenario_049,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"The reference applies 2026 OBBBA law. It starts from AGI of $247,385, which is wages minus only the $15,436 traditional 401(k), plus $3,501 of interest and $6,320 of dividends. It then subtracts the $32,200 MFJ standard deduction, the full $25,000 qualified overtime deduction (MAGI is below the $300,000 joint phase-out start), and the roughly $580 non-itemizer cash charitable deduction, leaving $189,605 of taxable income. Tax is $29,771 on $183,397 of ordinary income under the 10/12/22% brackets (thresholds $24,800, $100,800 and $211,400), plus 15% on the $6,208 of qualified dividends. Most wrong models either left out the overtime deduction or assumed the TCJA provisions expired in 2026. Others subtracted the employer-sponsored insurance premium from the stated wages, took the auto-loan interest deduction even though it fully phases out above $200k MAGI, or missed the non-itemizer charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"AGI is $247,385: wages minus only the $15,436 traditional 401(k), plus interest and dividends. The $16,408 ESI premium does not reduce wages, the $721 IRA deduction is phased out, and Roth contributions are not deductible. Under OBBBA rules for 2026, three deductions apply: the $32,200 standard deduction, the $25,000 joint-capped qualified overtime deduction, and the $579.76 non-itemizer cash charitable deduction. The auto-loan interest deduction is fully phased out above $200k MAGI. On the resulting $189,605.23 of taxable income, the 2026 joint brackets give $29,771.39, and 15% on $6,208 of qualified dividends adds $931.20. Most models missed the overtime deduction or used TCJA-sunset rules. The rest either excluded the ESI premiums from wages or got the charitable or auto-loan deduction wrong." us,scenario_049,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -419,7 +419,7 @@ us,scenario_049,head_medicaid_eligible,1,parse_contract_failure,missing_output,F us,scenario_049,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or a qualifying early-eligibility condition such as disability, end-stage renal disease, or ALS. The 53-year-old head has none of those conditions, so the eligibility result is No." us,scenario_049,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_049,payroll_tax,32,llm_error,payroll_tax_base,False,,"Only the spouse has wages: $253,000 in FICA wages, with nothing subtracted. Employee Social Security tax is 6.2% of the 2026 wage base of $184,500, or $11,439. Medicare tax is 1.45% of $253,000, or $3,668.50. Additional Medicare Tax is 0.9% of the $3,000 above the $250,000 joint threshold, or $27. Total: $15,134.50. Models missed the answer in two main ways. One group used a stale or guessed wage base (often the 2025 figure of $176,100). The other group subtracted the $16,408 employer-sponsored insurance premiums from FICA wages. That cuts Medicare tax and wrongly removes the $27 Additional Medicare Tax. A few models got $15,134.50 in their reasoning and then submitted a different number." +us,scenario_049,payroll_tax,33,llm_error,payroll_tax_base,False,,"The reference uses the spouse's full $253,000 gross wages as the FICA base, with no reduction for employer-sponsored insurance premiums. Social Security tax is 6.2% of the 2026 wage base of $184,500, which is $11,439. Medicare tax is 1.45% of $253,000, which is $3,668.50. Additional Medicare Tax is 0.9% of the $3,000 above the $250,000 joint threshold, which is $27. The total is $15,134.50. Most wrong models did one of two things. Some subtracted the $16,408 in premiums as pretax cafeteria-plan deductions, which lowered Medicare wages below $250,000 and dropped the $27 Additional Medicare Tax. Others capped Social Security at an older or guessed wage base such as $176,100 or $183,600. A few computed the correct total and then submitted a different number." us,scenario_049,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -431,23 +431,23 @@ us,scenario_049,ssi,1,parse_contract_failure,missing_output,False,,All wrong res us,scenario_049,state_income_tax_before_refundable_credits,9,llm_error,thresholds_rates,False,,"New Hampshire does not tax wages. Its only personal income levy, the Interest and Dividends Tax, produces no liability on this household in 2026. The trap is the rate schedule. The 2021 law cut the rate one point a year and would have left it at 1% in 2026. The 2023 law (HB 2) replaced that schedule and ended the tax for taxable periods beginning after December 31, 2024. PolicyEngine computes the same $5,021 base the models used ($9,821 of interest and dividends minus the $4,800 joint exemption) and assesses $0. Almost every wrong model found that base correctly and then applied a superseded nonzero rate: 1%, or in one case 3%." us,scenario_049,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_051,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Under 2026 post-OBBBA law, a single filer with $40,000 in wages gets the $16,100 standard deduction. That leaves $23,900 of ordinary taxable income, and tax on it is 10% × $12,400 + 12% × $11,500 = $2,620. The $180 of non-Schedule-D capital gain distributions is taxed at the 0% preferential rate, because total taxable income is far below the 2026 0% LTCG threshold, so it adds nothing. The largest group of models (answering $2,641.60) taxed that $180 at the 12% ordinary rate, which adds exactly $21.60. Other models used outdated standard deductions or brackets, assumed the TCJA had expired (personal exemption and a 15% bracket), subtracted employer insurance premiums from wages, or submitted numbers that their own arithmetic did not support." +us,scenario_051,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"Under the 2026 OBBBA parameters for a single filer, the standard deduction is $16,100 and the 10% bracket ends at $12,400. That leaves $23,900 of ordinary taxable income from the $40,000 of wages, taxed at $1,240 + 12% × $11,500 = $2,620. The $180 of non-Schedule D capital gains (capital gain distributions) adds no tax: it is either left out of gross income or taxed at the 0% long-term rate, well below the roughly $49,450 threshold. The most common wrong answer, $2,641.60, taxes that $180 at 12%. Most other wrong answers use outdated 2025 or pre-OBBBA deduction and bracket figures, assume the TCJA expired (bringing back the personal exemption and the 15% bracket), subtract ESI premiums that were never stated as pre-tax, or invent nonrefundable credits." us,scenario_051,federal_refundable_credits,4,llm_error,thresholds_rates,False,,"A childless single filer with approximately $40,000 of earnings is above the 2026 EITC phaseout endpoint, so the EITC is $0. With no qualifying children and no facts supporting another refundable federal credit, all five refundable-credit components are $0." us,scenario_051,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Louisiana Medicaid eligibility requires qualification through a specific categorical pathway, not merely acceptable immigration status or the absence of an expressly listed disqualification. This 25-year-old, nondependent adult has MAGI income of 2.51 times FPL, receives no SSI, and falls into Medicaid category NONE, so no MAGI or alternative state-program pathway confers eligibility." us,scenario_051,payroll_tax,10,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $40,000 of wages here, producing $2,480 and $580 respectively, for a $3,060 total. The dominant error was subtracting the listed employer-sponsored insurance premium without any fact establishing a pre-tax salary-reduction arrangement; several other models derived $3,060 correctly but then submitted a different number." us,scenario_051,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_051,state_income_tax_before_refundable_credits,47,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2024 reform (Act 11) set a flat 3% rate and a $12,500 single standard deduction for 2025, and that deduction is indexed for inflation starting in 2026. The 2026 single standard deduction is $12,835, so taxable income is $40,180 − $12,835 = $27,345 and the tax is $820.35. The most common wrong answer, $830.40, used the unindexed 2025 deduction of $12,500, which leaves $10.05 too much tax. Most of the other wrong answers used pre-2025 law: the 1.85%/3.5% brackets, the $4,500 exemption, and the federal income tax deduction. Some of those also subtracted the $5,389 ESI premiums from AGI ($34,791), even though Louisiana AGI is $40,180. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)." +us,scenario_051,state_income_tax_before_refundable_credits,48,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2025 reform set a flat 3% rate and a $12,500 single standard deduction, indexed from 2026 under R.S. 47:294(B). PolicyEngine computed $12,835 from published CPI-U, giving the retained reference of 3% × ($40,180 − $12,835) = $820.35. Louisiana's first published 2026 return deduction was $12,838 in RIB 26-019 on 2026-09-28, giving $820.26; this output is excluded under d994. Louisiana AGI equals federal AGI of $40,180 because the ESI premiums do not reduce wages here. The largest final-answer group, 22 of the 48 models, used $12,500 and answered $830.40 or $830; four more computed about $830 using that deduction and then submitted another amount. Fourteen used repealed rules, including the $4,500 exemption or deduction, the 1.85%/3.5% brackets, or the deduction for federal income tax; six of these also subtracted the $5,389 ESI premium from AGI. The remaining eight claimed no tax or an offsetting deduction, gave no calculation, or gave no answer. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)." us,scenario_051,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_052,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine deducts the head's $43.28 traditional IRA contribution above the line, which is the amount beyond the $3,000 capital-loss ALD in its $3,043.28 total, without applying the IRC §219(g) active-participant phase-out. The head's 401(k) elective deferrals make them an active participant, and joint MAGI of about $505k is far above the 2026 MFJ phase-out range ($129,000–$149,000), so the statutory deduction is $0 and the statutory liability is $104,225.30, $13.89 above the reference.","The reference applies the permanent 2026 OBBBA joint-filer parameters to taxable income of $472,578.56: a $32,200 standard deduction, no personal exemptions, and a 32% bracket running from $403,550 to $512,450. The only reductions are the $926.16 401(k) deferral, the $3,000 capital-loss limit, and the $43.28 traditional IRA contribution; ESI premiums do not reduce wages and Additional Medicare Tax is excluded. The total is $103,865.78 of ordinary tax, plus $127.20 on the $848 of qualified dividends at 15%, plus $218.42 of NIIT on $5,748 (interest plus dividends, net of the $3,000 allowable loss). Most models missed because they used the expired pre-TCJA schedule with personal exemptions, subtracted the $16,208 ESI premiums from wages, used the wrong standard deduction or brackets, left out or overstated NIIT, or added the $2,250 Additional Medicare Tax. The closest models differ only because they disallowed the $43.28 IRA deduction. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." +us,scenario_052,federal_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine takes the head's $43.28 traditional IRA contribution as an above-the-line deduction (AGI $504,778.56) without applying the IRC §219(g) active-participant phase-out ($129,000–$149,000 MAGI for 2026 MFJ). That phase-out removes the deduction entirely for a head who makes 401(k) elective deferrals and has about $504k of MAGI; without the deduction the value is $104,225.30.","The reference applies OBBBA's permanent 2026 parameters ($32,200 MFJ standard deduction; 24% bracket up to $403,550, then 32%) to taxable income of $472,578.56. Wages are reduced only by the $926.16 401(k) deferral and the $43.28 IRA deduction. Capital losses are capped at $3,000, and the $16,208 ESI premium is not subtracted. The engine then adds 15% tax on $848 of qualified dividends and 3.8% NIIT on $5,748 of net investment income, giving $103,865.78 + $127.20 + $218.42. Common gaps to the frozen reference involved assuming the TCJA sunset (personal exemptions and the old 28%/33% schedule), subtracting the ESI premium from wages, adding the Additional Medicare Tax to income tax, or differing from the reference's NIIT treatment; the exclusion record leaves whether this output includes NIIT unresolved. The closest answers ($104,225.30) differ only by disallowing the $43.28 IRA deduction. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_052,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_052,head_medicare_eligible,1,llm_error,age_disability,False,,"Standard Medicare eligibility requires age 65 unless a separate qualifying pathway, such as disability or end-stage renal disease, applies. The head is 59 and all unlisted disability and health-status facts are false, so no Medicare eligibility pathway applies." us,scenario_052,payroll_tax,32,llm_error,payroll_tax_base,False,,"Only the head earns wages. The reference applies three components to the full $500,000: 6.2% Social Security on the 2026 wage base of $184,500 ($11,439), 1.45% Medicare on all $500,000 ($7,250), and 0.9% Additional Medicare Tax above the $250,000 married-filing-jointly threshold ($2,250), for $20,939. Two errors account for most misses. Some models used an outdated or guessed Social Security wage base, such as $168,600, $176,100 or about $180,000. Others shrank Medicare wages by subtracting the employer-sponsored insurance premium or the traditional 401(k) deferral, even though the facts never describe a cafeteria plan and 401(k) deferrals stay subject to FICA. Other models used the $200,000 single threshold, made arithmetic slips, or submitted numbers that did not match their own work." us,scenario_052,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or an applicable disability or other qualifying pathway. The 55-year-old spouse has no listed disability or other Medicare-qualifying status, so the spouse is not eligible." -us,scenario_053,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The OBBBA made the TCJA structure permanent, so 2026 law uses a $16,100 single standard deduction, no personal exemption, and brackets of 10% up to $12,400, 12% up to $50,400, and 22% above that. The household's gross wages are AGI because the employer-insurance premium input does not reduce them, and the OBBBA auto-loan-interest deduction does not apply because no qualifying new-vehicle facts are given. That leaves taxable income of $50,868.68 and tax of $1,240 + $4,560 + 22% × $468.68 = $5,903.11. The models went wrong in three main ways. Some assumed the TCJA would sunset and restored the personal exemption and the 15%/25% brackets. Some used 2023–2025 or guessed 2026 amounts for the deduction and bracket thresholds. Others took $3,389 of ESI premiums (or the $950 of auto loan interest) off AGI. A few computed roughly the right figure and then submitted a different number." +us,scenario_053,federal_income_tax_before_refundable_credits,36,llm_error,thresholds_rates,False,,"This calculation uses 2026 law after OBBBA, which kept the TCJA structure permanently. AGI equals the full $66,968.68 in wages, with no subtraction for employer-plan premiums and no deduction for auto loan interest. The $16,100 single standard deduction leaves $50,868.68 of taxable income. The 2026 brackets (10% up to $12,400, 12% up to $50,400, 22% above) give $1,240 + $4,560 + $103.11 = $5,903.11, and no nonrefundable credits apply. The models went wrong in three main ways: they assumed the TCJA had expired (adding personal exemptions and 15%/25% brackets), they used 2024 or 2025 thresholds, or they subtracted the $3,389 in ESI premiums or the $950 of auto loan interest from income. Several models also worked out a number near the reference and then submitted a different one." us,scenario_053,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"A single 25-year-old filer with no children and $66,969 of wages receives neither a refundable Child Tax Credit nor an Earned Income Tax Credit in 2026. With the other enumerated refundable-credit components also equal to zero, federal refundable credits total $0." us,scenario_053,payroll_tax,10,llm_error,payroll_tax_base,False,,"PolicyEngine applies the employee FICA rates to the full $66,969 of gross wages: 6.2% Social Security ($4,152.06) plus 1.45% Medicare ($971.05) equals $5,123.10. The listed employer-sponsored insurance premium ($3,389) does not reduce the payroll-tax wage base. The household facts never say the premium is paid through a Section 125 cafeteria plan, and the prompt forbids inferring unlisted facts. Eight models subtracted the premium anyway and taxed $63,580, which gives $4,863.87. The two GPT-5.4 models used the correct base and rates but multiplied them out wrong." us,scenario_053,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_053,state_income_tax_before_refundable_credits,47,reference_engine_defect,state_local_rule,False,,"Idaho taxable income here is federal AGI of $66,968.68 (wages are not reduced by the ESI premium) minus the $16,100 standard deduction, which gives $50,868.68. Idaho's 2026 schedule taxes the first $4,920 of a single filer's taxable income at 0% and everything above it at a flat 5.3%: 5.3% × $45,948.68 = $2,435.28, and no nonrefundable credits apply. Most models left out the indexed zero bracket, which pushes them to about $2,696. Others used the old 5.695%/5.8% rates or progressive brackets, used an outdated standard deduction ($15,000, $15,750 or $8,300), or subtracted the $3,389 ESI premium from AGI. The closest answers missed only because they used the 2025 or a misestimated zero-bracket amount instead of $4,920. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)." +us,scenario_053,state_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"The reference uses four Idaho-specific steps. First, Idaho subtracts the $5,000 of health insurance premiums the household pays, which brings Idaho AGI to $61,968.68. Next, the $16,100 conforming standard deduction leaves taxable income of $45,868.68. Idaho then taxes that income at its 2026 flat rate of 5.3%, but only above the roughly $4,811 single zero-rate bracket, which gives $2,176.06. Finally, it adds the $10 permanent building fund tax, for a total of $2,186.06. Most models multiplied a flat rate (often the outdated 5.695% or 5.8%) by $50,869 or by an AGI reduced for employer-sponsored insurance (ESI) premiums. They missed the zero bracket, the $5,000 Idaho health-insurance subtraction, or both. Even the closest model left out the $10 building fund tax." us,scenario_053,state_refundable_credits,40,llm_error,state_local_rule,False,,"Idaho's grocery credit is a refundable per-person income tax credit with no income limit or phase-out. For 2026, after the 2025 increase (HB 231), the amount is $155 for a resident under 65. This full-year, 25-year-old resident who received no SNAP qualifies for all 12 months, so state refundable credits equal $155. About half the models said $0, either calling the credit nonrefundable or inventing an income limit. Most of the rest knew it was refundable but used an older amount: $100, $120, or $150." us,scenario_054,child1_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; satisfying a CHIP income ceiling alone is not sufficient. At this very low household income, the 10-year-old qualifies for Medicaid under North Carolina’s OLDER_CHILD category, and that Medicaid eligibility excludes the child from CHIP." us,scenario_054,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Child 1 qualifies through the ordinary children's MAGI pathway: at age 10 he falls in the OLDER_CHILD category (ages 6–18), whose North Carolina income limit is 133% FPL plus the 5-percentage-point disregard, and NC folded its separate NC Health Choice tier into Medicaid so school-age children are covered under Medicaid rather than being pushed to CHIP. The household's only income is $17,100 of self-employment earnings, which after the deductible half of SE tax leaves MAGI at 0.58 × FPL for a family of three — a fraction of the applicable limit. Both models returned 0 without ever performing that categorical-plus-income test: one asserted an income failure that the arithmetic contradicts, the other treated unlisted household facts as missing prerequisites even though the prompt instructs that unlisted booleans are false and take-up is assumed." @@ -465,7 +465,7 @@ us,scenario_054,head_wic_eligible,17,llm_error,categorical_eligibility,False,,"W us,scenario_054,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages, not SSTB self-employment income; that income instead enters the separate self-employment-tax calculation. With no reported wages and all unlisted numeric inputs set to zero, every component of the requested employee-side payroll tax is zero." us,scenario_054,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. The household is in the FREE tier because it meets categorical eligibility through SNAP or TANF and its school-meal income is 63% of the federal poverty guideline, below the 130% free-meals threshold; therefore reduced-price support is zero." us,scenario_054,self_employment_tax,7,llm_error,other,False,,"This is a plain SECA calculation. Net SE earnings are $17,100 × 0.9235 = $15,791.85, which is far below the Social Security wage base, so the full 15.3% applies: $15,791.85 × 0.153 = $2,416.15. Every model either picked the right method and then made an arithmetic or rounding slip, or worked out about $2,416 and then submitted some other number. Two models subtracted things that do not reduce SE tax at all, such as the deduction for half of SE tax, which only lowers income-tax AGI." -us,scenario_054,snap,45,llm_error,thresholds_rates,False,,"The household has $1,425/month of gross self-employment earnings. Its net income is $931 after the $285 earned income deduction (20%) and the FY2026 $209 standard deduction for three people. The expected contribution is 30% of $931, rounded up to $280, and the $785 three-person maximum minus $280 gives $505/month, or $6,060 a year. The models split into four groups of errors: (1) models that used the right inputs but did not round the contribution up, getting $505.70 ($6,068.40) or $506 ($6,072); (2) models that used stale or invented maximum allotments and standard deductions from FY2024, FY2025, or rough projections; (3) models that wrongly reduced self-employment income by SE tax or by a 40% cost allowance; (4) models that zeroed out the benefit by applying a resource limit, which North Carolina's broad-based categorical eligibility through TANF non-cash eligibility waives." +us,scenario_054,snap,46,llm_error,thresholds_rates,False,,"The household gets $505 a month: the $785 FY2026 maximum allotment for three people, minus an expected contribution of $280. That contribution is 30% of $931 net income ($1,425 gross, less the $285 earned-income deduction and the $209 standard deduction), which is $279.30, rounded up to the next whole dollar. The models that came closest used the right inputs but subtracted the unrounded $279.30 (or rounded the benefit instead), getting $505.70 or $506. Most of the others used FY2025 or made-up values for the maximum allotment or standard deduction. Several applied a resource limit that NC's broad-based categorical eligibility (via TANF non-cash eligibility) removes, and a few subtracted SE tax or an assumed business-cost share, even though SNAP counts the full $17,100 when no expenses are listed." us,scenario_054,state_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Two independent paths lead to $0 of North Carolina tax. First, the only income fact listed is the SSTB *component* input; total self-employment income is unlisted and therefore 0 under the prompt's ""treat any unlisted numeric input as 0"" rule, so the engine's trace shows essentially no income entering nc_income_tax. Second, even crediting the full $17,100 as net self-employment earnings, federal AGI is $17,100 − $1,208 (half of SE tax) = $15,892, and North Carolina's 2026 head-of-household standard deduction of $19,125 — the head has two qualifying children — exceeds that, driving NC taxable income to $0 and the 3.99% 2026 flat rate onto a zero base. Every wrong model produced a positive liability; two of them derived exactly $0 in their own reasoning and then submitted a different number anyway." us,scenario_055,federal_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"The 2026 result follows OBBBA law. AGI is $156,275.86, with 85% of Social Security taxable. The filer takes the $18,150 standard deduction ($16,100 base plus $2,050 aged add-on). The $6,000 senior deduction phases down by 6% of MAGI above $75,000, leaving $1,123.45. The new §170(p) charitable deduction for non-itemizers adds $1,000 for a single filer. That leaves taxable income of $136,002.41, taxed at the OBBBA-reset brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, then 24%). The closest models handled everything except the $1,000 non-itemizer charitable deduction and came out exactly $240 high. A large group applied TCJA-sunset law (personal exemption, 15/25/28% brackets), which OBBBA repealed. Others left out the senior deduction, did not phase it out, or used inflation-projected parameters." us,scenario_055,payroll_tax,5,llm_error,thresholds_rates,False,,"The only FICA-taxable income here is the $83,635 of wages: the $67,760 taxable pension and $5,742 Social Security retirement benefit are outside the payroll tax base, the wage is far below the 2026 Social Security wage base, age 70 does not exempt wages from FICA, and Florida levies no mandatory employee state payroll tax. The answer is therefore exactly 6.2% × $83,635 = $5,185.37 plus 1.45% × $83,635 = $1,212.71, or $6,398.08. What separates the reference from the wrong answers is execution of that single multiplication at the employee-side rate: two models stated the right rule and then mis-multiplied or rounded, one submitted a number contradicting its own correct explanation, one used the combined 12.4%/2.9% employer-plus-employee rates before guessing, and one submitted a placeholder zero." @@ -475,22 +475,22 @@ us,scenario_056,head_wic_eligible,1,parse_contract_failure,missing_output,False, us,scenario_056,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_056,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage earnings, and the household has no reported wages or salary. The listed SSTB self-employment income belongs in the separately requested self-employment-tax calculation, while alimony and financial assistance are not employee FICA wages." us,scenario_056,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax applies because $4,340 exceeds the $400 filing threshold. Multiplying the 92.35% net-earnings base by the combined 15.3% Social Security and Medicare rate yields $613.22; the threshold does not reduce the taxable base or eliminate liability." -us,scenario_056,snap,46,prompt_ambiguity,categorical_eligibility,False,,"Gross income counts the $12,000 financial assistance as unearned income, giving $1,486.54/month: $361.67 self-employment, $124.92 alimony and $1,000 assistance. That is 112% of FPL, well under NJ's broad-based categorical eligibility limit. After the $213.68 standard deduction and the $72.33 earned-income deduction, adjusted income is $1,200.53. Shelter costs are only the $995.75/month real estate taxes. Home mortgage interest is a tax-deduction input and is not a SNAP shelter cost here, so the excess shelter deduction is an uncapped $395.49 and net income is $805.04. The formula benefit is $304.68 − $241.50 = $63.18, which New Jersey's $95 state minimum benefit replaces, giving $1,140 a year. Models went wrong in four ways: they excluded the assistance, counted mortgage interest and hit the shelter cap, declared the household ineligible, or zeroed out a sub-maximum formula without applying the NJ $95 floor. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)." +us,scenario_056,snap,47,prompt_ambiguity,categorical_eligibility,False,,"The $1,140 annual amount is a frozen reference excluded from scoring because work hours and occupied-home mortgage status are unlisted. Its historical derivation assumed 40 weekly work hours and property-tax-only shelter costs, producing a $63.18 formula benefit raised to New Jersey's $95 monthly minimum. Mortgage payments including interest count as SNAP shelter costs if the household occupies the mortgaged home. Some $0 answers used the zero-hours ABAWD reading recognized by the developer decision. The sidecar's 2026-09-29 recheck reproduces $1,908/yr with 40 weekly hours and mortgage interest counted, and $95/yr under the engine's zero-hours and county defaults; later rechecks retain $95 while preserving the frozen $1,140 and exclusion. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)." us,scenario_056,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_056,state_income_tax_before_refundable_credits,5,llm_error,thresholds_rates,False,,"New Jersey's gross income tax exempts a single filer entirely when New Jersey gross income falls at or below the $10,000 minimum filing threshold, and this household's NJ gross income is $5,838.53 (alimony $1,498.53 plus self-employment $4,340.00; the $12,000 of financial assistance is not NJ gross income). PolicyEngine therefore computes the bracket tax as an intermediate figure — $4,838.53 of taxable income after the $1,000 regular exemption, times the 1.4% first-bracket rate, equals $67.74 — and then zeroes it out under the below-threshold exemption, leaving $0.00. Every wrong model stopped at the 1.4% bracket arithmetic and never applied the threshold exemption; three of them also mis-specified the NJ income and deduction base along the way." -us,scenario_056,state_refundable_credits,47,llm_error,credit_phaseout,False,"PolicyEngine's NJ EITC formula for childless filers aged 18 to 24 caps the unscaled $308.55 federal-formula phase-in amount at 40% of the $664 2026 federal maximum ($265.60). N.J.S.A. 54A:4-7 instead requires 40% of the full federal-formula credit, which gives 0.40 × $308.55 = $123.42.","The deciding rule is that New Jersey gives its refundable EITC to filers aged 18 and over who have no qualifying children. It is computed with the federal formula even though the federal childless credit requires age 25. Most models tied NJ eligibility to federal eligibility or to an outdated age-21 floor, and returned $0 or a $50 property-tax credit. EITC earned income here is $4,340 minus half of SE tax ($306.61), which is $4,033.39, so the phase-in amount is $308.55. Models that applied NJ's 40% match to that amount got $123.42. The reference's $265.60 equals 40% of the $664 2026 maximum childless credit, which the engine reaches by capping the $308.55 phase-in amount at that scaled maximum. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)." +us,scenario_056,state_refundable_credits,48,llm_error,credit_phaseout,False,"The engine's $265.60 equals 40% of the full $664 2026 federal childless EITC maximum, not 40% of the $308.55 phased-in amount shown in its own trace. N.J.S.A. 54A:4-7 sets the credit for a childless filer aged 18–24 at 40% of the federal-formula credit, so it cannot exceed 7.65% × $4,033.39. The likely cause is a phase-out reduction computed on the negative AGI (−$5,882.05) that is not floored at zero, which pushes the credit up to the $664 cap.","The case turns on New Jersey extending its 40% EITC match to childless filers aged 18–24. NJ computes that credit as the federal-formula EITC the filer would get without the federal age-25 minimum. Most models applied the federal age test and returned $0. Models that applied the NJ age-18 rule computed 40% × 7.65% × $4,033.39 of net self-employment earnings = $123.42, while the engine's $265.60 equals 40% × the $664 2026 childless maximum, the amount its reduction step produced on the −$5,882.05 AGI. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)." us,scenario_056,tanf,2,llm_error,categorical_eligibility,False,,"TANF is categorically limited to assistance units containing a dependent child: New Jersey's WorkFirst NJ/TANF requires a needy child under 18 (or 18 and a full-time secondary student) in the unit, and childless adults are served instead by WFNJ General Assistance, which is not TANF. This household is a single 20-year-old head with no children, so the benefit is $0 regardless of the income test. The second trap is that `financial_assistance` is an input income line in the household facts, not a computed program output; echoing it back as the TANF benefit both invents categorical eligibility and confuses an input with the requested computed value." us,scenario_057,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The household's only refundable credit is the no-qualifying-child EITC, and it gets the full 2026 maximum of $664. The joint return passes the age test because one spouse, the head at 43, is between 25 and 64; only one spouse has to qualify. Earned income is about $9,665: $10,400 of self-employment income minus the deductible half of self-employment tax. That is above the 2026 phase-in ceiling of about $8,680 and well below the married-filing-jointly phase-out start of about $18,000, so the credit sits on the plateau. Models split two ways. Some denied the credit outright, citing the age test, treating self-employment income as non-earned, or claiming the household was in the phase-out. The rest found the plateau but used a stale or mis-indexed maximum ($649 for 2025, or $666–$667) or made arithmetic and phase-out errors." us,scenario_057,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"The head is blind and disabled, with very low countable income ($10,400 of self-employment income, $14 of disability benefits) and only $912 in assets. That makes the head an SSI recipient, receiving about $7,231 a year. Louisiana gives automatic Medicaid to SSI recipients, so the engine puts the head in the SSI_RECIPIENT category. The head would also qualify under Louisiana's adult expansion, because household MAGI is only 0.45× FPL, far below the 138% limit. All four models missed the SSI pathway, and none of them checked the head's income against the expansion limit correctly." us,scenario_057,head_medicare_eligible,10,llm_error,age_disability,False,,"Before 65, a person qualifies for Medicare only after 24 months of Social Security disability (SSDI) entitlement, or through the ESRD/ALS pathways. PolicyEngine checks this with months of Social Security disability receipt, and it does not use the is_disabled or is_blind flags. The head is 43. Their only disability income is $14 of generic ""disability benefits"", which is not Social Security disability. No months of SSDI receipt are listed, so that input is 0, and neither ESRD nor ALS is listed either. Every model treated disability status plus a nominal disability-benefit amount as enough for Medicare under 65. That skips both the SSDI-entitlement requirement and the 24-month waiting period." us,scenario_057,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_057,self_employment_tax,7,llm_error,thresholds_rates,False,,"The whole case reduces to one canonical computation: net earnings from self-employment are 92.35% of the $10,400 SSTB profit ($9,604.40), and the combined OASDI/HI rate of 15.3% applies in full because $9,604.40 is far under the 2026 Social Security wage base, giving 0.153 × 9,604.40 = $1,469.47. The $400 figure in §1402(b) is the floor at which SE tax begins, not a ceiling below which it is waived, and no other floor exists; the one-half-of-SE-tax figure is an above-the-line deduction under §164(f), not the liability. The prompt further defines this output as SE tax excluding Additional Medicare Tax, and the 0.9% AMT threshold for joint filers is $250,000 of combined earnings, so nothing is added on top. Every wrong answer comes from inverting the $400 floor, inventing a different floor, truncating the 0.9235 base factor, reporting the deduction instead of the tax, or bolting on a nonexistent AMT." -us,scenario_057,snap,47,prompt_ambiguity,age_disability,False,,"The trap is the SSI amount, which must be counted as unearned income for SNAP. The engine gives the head $602.58/month of SSI: the individual rate of $994 minus $391.42 of countable income ($866.67 of self-employment income, minus the leftover $18.83 general exclusion and the $65 earned-income exclusion, then halved). Monthly SNAP gross income is therefore $866.67 + $3.17 + $602.58 = $1,472.42. Subtracting the 20% earned-income deduction ($173.33) and the $209 standard deduction leaves net income of $1,090.08, so the benefit is $546 − $327 = $219/month for January–September. October–December use FY2027 parameters ($558.24 − $325.50 = $232.74/month), for a total of $2,669.22. Roughly half the models left SSI out of SNAP income and got about $4,700. Most of the others counted SSI at the couple rate of about $1,099/month and got about $840. The rest called the household ineligible or produced unsupported numbers. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." +us,scenario_057,snap,48,prompt_ambiguity,age_disability,False,"PolicyEngine's SSI of $602.58/month equals the $994 individual FBR minus only the head's $391.42 countable income, so the disabled 20-year-old spouse was treated as SSI-ineligible; under 42 U.S.C. 1382(b)(2) and 20 CFR 416.412, a married couple who are both disabled, with $912 in resources, is an eligible couple at the $1,491 couple FBR, which gives SSI of about $1,098.58/month and SNAP of about $880/year.","PolicyEngine counts SSI of $602.58/month (the $994 individual FBR minus $391.42 of the head's countable self-employment earnings) as unearned SNAP income. Adding $866.67 of self-employment income and $3.17 of disability benefits gives gross income of $1,472.42. Subtracting the 20% earned-income deduction ($173.33) and the $209 standard deduction leaves net income of $1,090.08, so the benefit is $546 − $327 = $219 for January–September and $558.24 − $325.50 = $232.74 for October–December under FY2027 parameters, totaling $2,669.22. Most wrong answers either left out SSI (about $4,700–4,800), counted SSI at the roughly $1,099 eligible-couple rate (about $840), or wrongly denied eligibility outright. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." us,scenario_057,spouse_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires the person to be outside Medicaid eligibility; a low income alone does not establish CHIP eligibility. The spouse is Medicaid-eligible in Louisiana's adult category, so the Medicaid exclusion makes `spouse_chip_eligible` false without applying a CHIP income threshold." us,scenario_057,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Louisiana adopted the ACA adult expansion effective July 1, 2016, so any non-elderly adult aged 19–64 who is not otherwise eligible qualifies for Medicaid on income alone at or below 138% FPL under MAGI methodology — no pregnancy, parenthood, or disability category is required. The spouse is 20 with household MAGI at 0.45 x FPL (the only income is the head's $10,400 SSTB self-employment plus $38 of disability benefits, against a two-person 2026 FPL near $21,150 and a 138% cutoff near $29,200), placing the household roughly a third of the way to the expansion limit. Every wrong model either invented an income excess that does not exist or applied pre-ACA categorical gatekeeping that Louisiana abolished, and each therefore returned 0 where the ADULT expansion pathway returns eligible." us,scenario_057,spouse_medicare_eligible,8,llm_error,age_disability,False,,"To be eligible for Medicare before age 65, a person must be entitled to Social Security Disability Insurance (SSDI) and have received it for 24 months, or have ESRD or ALS. Being disabled is not enough on its own. The spouse is 20 and has only $24 of generic `disability_benefits`. That is private or other disability income, not SSDI. Months of SSDI receipt and ESRD/ALS status are not listed, so they default to 0/false. The spouse fails both the age-65 test and the disability pathway. Every model read ""is disabled"" plus a small disability-benefit amount as satisfying the disability pathway." us,scenario_057,spouse_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC does not cover women merely because they are young, disabled, or income-eligible. A woman must be pregnant, breastfeeding, or postpartum; the spouse has none of these statuses, and the household has no infants or children under age five." -us,scenario_057,ssi,47,prompt_ambiguity,age_disability,False,,"Under PolicyEngine's rules, only the head passes the SSI aged/blind/disabled test. The spouse has is_ssi_aged_blind_disabled=False, so the head is paid the individual federal benefit rate of $994/month as an eligible person with an ineligible spouse, not the $1,491 couple rate. The spouse's $2/month of unearned income is below the $497 deeming threshold (the couple rate minus the individual rate), so nothing is deemed. The head's own $1.17/month of unearned income uses up part of the $20 general exclusion. The remaining $18.83 and the $65 earned-income exclusion come off $866.67 of monthly earnings, and half of what is left is countable: $391.42. The benefit is $994 − $391.42 = $602.58/month, or $7,231/year. Most models paid an eligible-couple benefit of about $13,100. The $0 answers either skipped the one-half earned-income exclusion or wrongly treated earnings, assets, or the absence of listed SSI receipt as disqualifying. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." +us,scenario_057,ssi,48,prompt_ambiguity,age_disability,False,,"The engine counts only the head (age 43, blind) as SSI aged/blind/disabled. It treats the 20-year-old spouse as an ineligible spouse, so the head is paid as an eligible individual at the 2026 individual FBR of $994/month, not the eligible-couple rate of about $1,491/month. The spouse's $24/yr of unearned income is too small to produce deemed income. The head's own countable income is $391.42/month: the $1.17/month disability benefit is absorbed by the $20 general exclusion, the remaining $18.83 and the $65 earned exclusion come off the $866.67/month of self-employment earnings, and the rest is halved. That gives $994 − $391.42 = $602.58/month, or $7,231/yr. Most models applied the couple FBR and got about $12,400–$13,600. The models that answered zero skipped the one-half earned-income exclusion, misread the resource or take-up facts, or gave no computation. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." us,scenario_057,state_income_tax_before_refundable_credits,3,llm_error,state_local_rule,False,,"The trap is Louisiana's flat-tax reform, effective tax year 2025: Act 11 of the 2024 Third Extraordinary Session replaced the graduated 1.85/3.5/4.25% schedule and the $4,500-per-filer personal exemptions with a single 3% rate and a $25,000 standard deduction for married filing jointly (indexed from 2026), and Louisiana taxable income starts from federal AGI. This household's only taxable income is $10,400 of SSTB self-employment income, yielding federal AGI of about $9,665 after the deductible half of self-employment tax — far below the joint standard deduction — so Louisiana taxable income and tax are both zero. Both wrong answers subtracted $9,000, the repealed exemption figure, manufacturing a $665 taxable base that the current deduction fully absorbs." us,scenario_057,state_refundable_credits,35,llm_error,state_local_rule,False,,"This joint-filing couple with no children qualifies for the federal childless EITC. The head is 43, and on a joint return only one spouse has to be 25–64. The household's earned income is about $9,665 ($10,400 of self-employment income minus half of the roughly $1,469 SE tax). That is above the childless phase-in ceiling and below the joint-return phase-out start, so they get the full 2026 maximum of $664. Louisiana's refundable EITC is 5% of the federal EITC, or $33.20. Most models wrongly said Louisiana has no refundable EITC, or no income tax at all, or set the federal EITC to zero. The rest used the wrong year's federal maximum or the wrong state match rate." us,scenario_059,federal_income_tax_before_refundable_credits,13,llm_error,taxable_income_or_deductions,False,,"Provisional income is $25,600 + $18,552 = $44,152, only $152 above the $44,000 MFJ second-tier threshold. Taxable Social Security is therefore $6,000 + 0.85 × $152 = $6,129.20, which makes AGI $31,729.20. For 2026, OBBBA made the TCJA standard deduction permanent: $32,200 for MFJ plus $1,650 for each spouse aged 65+, for a total of $35,500. Each spouse also gets the new $6,000 senior deduction, adding $12,000. Total deductions of about $47,500 are far above AGI, so taxable income and tax are $0. Most models went wrong in one of two ways: they assumed the TCJA would sunset and used the pre-TCJA standard deduction plus personal exemptions (about $29–30.5k), or they overstated taxable Social Security by applying 85% to all benefits instead of using the tiered formula." @@ -499,7 +499,7 @@ us,scenario_060,federal_refundable_credits,1,llm_error,age_disability,False,,"Fo us,scenario_060,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Texas Medicaid does not grant eligibility merely because a 70-year-old has MAGI of 1.08 times FPL or minimal bank assets. The head must qualify through a specific Medicaid category, and the aged pathway is not satisfied; the engine therefore assigns Medicaid category NONE." us,scenario_060,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee-side FICA on $3,000 of wages is $186.00 of Social Security tax at 6.2% plus $43.50 of Medicare tax at 1.45%, totaling $229.50. The 2.9% Medicare rate combines the employee and employer shares, while the requested payroll-tax output excludes employer taxes; no additional Texas employee payroll tax applies." us,scenario_060,snap,4,llm_error,taxable_income_or_deductions,False,,"The trap is the SNAP treatment of the $45,600 in veterans benefits: 7 CFR 273.9(b)(2)(ii) counts veterans' benefits as unearned income, making countable gross income $62,775 rather than the $17,175 three models used. The elderly household skips the gross income test but must still pass the net income test, and even after the 20% earned income deduction, the standard deduction, the uncapped excess medical deduction over $35/month, and the uncapped shelter deduction available because the head is 70, net income lands near $45,000/year (~$3,760/month) against a one-person net limit around $1,300/month. Failing the net income test zeroes the allotment entirely — there is no minimum-benefit floor for an income-ineligible household — which is how the reference reaches $0. Three models excluded the VA income and therefore hit $0 net income and the one-person maximum allotment; the fourth counted it, derived ineligibility correctly, and then submitted a positive number anyway." -us,scenario_062,federal_income_tax_before_refundable_credits,25,prompt_ambiguity,taxable_income_or_deductions,False,,"The separate $28,800 survivor-benefits input is not part of federal gross income. AGI is farm rent $4,793 + interest $257 + pension $11,280 + taxable Social Security of $993 = $17,323. The $993 comes from provisional income of $16,330 + $10,656 = $26,986, which exceeds the $25,000 single base by only $1,986, and half of that is taxable. The head files as single because surviving-spouse (QSS) status requires a dependent child. The 2026 OBBBA-era $16,100 standard deduction, the $2,050 aged addition and the $6,000 senior deduction exceed AGI, so taxable income and tax are $0. Most models counted the $28,800 as taxable, which inflated provisional income enough to make 85% of Social Security taxable. The few models that got AGI right instead applied TCJA-sunset rules (a small standard deduction plus a personal exemption), and those rules left a small positive tax. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 4,328.39. The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72. Reference depends on an unlisted input; output excluded from scoring (taxability of survivor_benefits)." +us,scenario_062,federal_income_tax_before_refundable_credits,26,prompt_ambiguity,taxable_income_or_deductions,False,,"The engine's income base is $37,642: farm rent $4,793, interest $257, pension $11,280 and Social Security $21,312. The separate $28,800 survivor-benefits input is not in gross income. Non-SS income of $16,330 plus half of SS gives provisional income of $26,986, so at most $993 of SS is taxable and AGI is about $17,323. Under 2026 OBBBA law, the $16,100 single standard deduction, the $2,050 aged add-on and the $6,000 senior deduction ($24,150 in total) exceed that AGI, so taxable income and tax are $0. Most models added the $28,800 to AGI, which pushed 85% of SS into income. The models that left it out assumed the TCJA had expired and used smaller pre-TCJA deductions, which left positive taxable income. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 4,328.39. The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72. Reference depends on an unlisted input; output excluded from scoring (taxability of survivor_benefits)." us,scenario_062,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_062,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"The single trap is that an 85-year-old disabled Floridian has exactly one non-MAGI route to Medicaid in the engine — SSI receipt — and SSI is $0 here because unearned income of about $37,600 (Social Security survivor benefits $21,312, private pension $11,280, farm rent $4,793, taxable interest $257) is more than three times the ~$12,000 SSI federal benefit rate, and $29,100 in bank assets exceeds the $2,000 SSI resource limit. The MAGI route is closed twice over: Florida did not adopt the ACA adult expansion, and MAGI income sits at 2.36 x FPL, above every MAGI threshold. That leaves medicaid_category = NONE. All six models manufactured a yes by importing a pathway outside the eligibility determination — medically needy share-of-cost spend-down, Medicare Savings Programs, or spousal/institutional protections — or by excluding Social Security survivor benefits from SSI-related countable income, where they count in full past the $20 general disregard." us,scenario_062,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -512,14 +512,14 @@ us,scenario_064,child1_medicaid_eligible,9,llm_error,thresholds_rates,False,,"Th us,scenario_064,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,child1_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC eligibility requires both a categorical status and an income test, and this household fails both. Categorically, the ""child"" category covers children from their first birthday only until their fifth birthday; Child 1 is 12 and falls outside every WIC category (pregnant, postpartum, breastfeeding, infant, child under 5). Independently, the income test caps countable household income at 185% of the federal poverty guideline, and this household's roughly $98,810 of income for a 5-person unit is far above that ceiling. Both gates independently force is_wic_eligible = False." us,scenario_064,dependent1_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent1_medicaid_eligible,31,prompt_ambiguity,household_unit_or_filing_status,False,,"The engine places dependent1 (age 27, no income of their own) in the MAGI adult category. It measures their Medicaid income at 0.00× FPL, so the head's wages, 401(k) distributions and partnership income are not counted against them. With zero countable MAGI, dependent1 is far below Wisconsin's adult income limit. The MAGI adult category has no asset test, and having employer coverage does not bar Medicaid, so the answer is eligible. Nearly every model assumed the parents' roughly $100k+ tax-unit income counted toward dependent1. Several also wrongly applied SSI-related asset limits or treated the ESI as disqualifying. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." +us,scenario_064,dependent1_medicaid_eligible,32,prompt_ambiguity,household_unit_or_filing_status,False,,"The engine places Dependent 1 (age 27, under 65) in the MAGI-based adult category and measures their Medicaid income level from their own income, which is zero. That gives 0.00 x FPL, well under the adult limit, so they are eligible. Almost every wrong model gave the 27-year-old the head's roughly $100k+ tax-unit MAGI, or applied an ABD/SSI-related asset test that does not apply to MAGI categories. None of them checked the adult category against Dependent 1's own zero income. Having employer-sponsored insurance also does not bar Medicaid eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." us,scenario_064,dependent1_medicare_eligible,6,llm_error,age_disability,False,,"A person under 65 qualifies for Medicare only after 24 months of entitlement to Social Security Disability Insurance (SSDI), or through end-stage renal disease (ESRD) or ALS. Having a disability flag alone does not qualify anyone. Dependent 1 is 27. The household lists no SSDI or other Social Security benefits and no months of SSDI receipt, and all unlisted numeric inputs are 0, so Dependent 1 has no path to Medicare. The five models that answered treated the ""is disabled"" flag as Medicare eligibility and skipped the SSDI-entitlement step. The sixth model gave no usable answer." us,scenario_064,dependent1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent2_chip_eligible,10,llm_error,health_coverage,False,,"Dependent 2 fails CHIP on two independent grounds. First, Dependent 2 is enrolled in employer-sponsored insurance, and CHIP excludes a child already covered by a group health plan. Second, household income is 321.5% of the federal poverty guideline, above Wisconsin's 306% CHIP limit. Every model that answered checked only the age-under-19 test and an income test it had underestimated (273-300% FPL), and none applied the rule that existing ESI coverage disqualifies the child. Dependent 2 is also not Medicaid-eligible, so no Medicaid pathway rescues the answer." us,scenario_064,dependent2_medicaid_eligible,6,prompt_ambiguity,household_unit_or_filing_status,False,,"Dependent 2 is 18, so the only pathway open is the older-child MAGI category. It is income-tested against the five-person tax household's MAGI, which the engine puts at 3.22× FPL (322%). That is above Wisconsin's Medicaid limit for ages 6–18, above the roughly 306% BadgerCare/CHIP ceiling the models quoted, and above that ceiling plus the 5-point disregard (about 311%). No other pathway applies: there is no disability, pregnancy, caretaker status or SSI. Every model that answered Yes assumed or computed household MAGI below about 300–306% FPL and treated the CHIP ceiling as the Medicaid child limit. Its income measure fell short of the engine's 322%. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 0.00; the alternative is 1.00. The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household. Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." us,scenario_064,dependent2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent2_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"The reference starts with AGI of $114,421.41. That is wages net of the $2,315 401(k) deferral, plus the $19,350 partnership/S-corp income (the $14,000 self-employment partnership amount is part of it, not added on top), plus the $20,000 distribution, minus the $16,800 farm loss, minus $3,000 of capital loss, minus the $108 IRA contribution. It then subtracts the 2026 OBBBA $32,200 standard deduction, the $13,899 FLSA overtime deduction and a $510 QBI deduction (20% of $19,350 − $16,800). That gives $67,812.12 taxable income and $7,641.46 of tax; the $2,200 CTC and two $500 other-dependent credits leave $4,441.46. Because the farm loss more than offsets the $14,000 SE partnership income, there is no SE tax or half-SE deduction, and the domestic production deduction was repealed after 2017. The wrong answers mostly went astray in a few ways: they left out the overtime deduction, counted the $14,000 twice, took the repealed DPAD or a half-SE-tax deduction, applied the pre-TCJA sunset rules, or added amounts that are not part of this measure." +us,scenario_064,federal_income_tax_before_refundable_credits,45,llm_error,taxable_income_or_deductions,False,,"The correct path starts from AGI of $114,421. That figure comes from wages less the $2,315 traditional 401(k) deferral, plus the $19,350 partnership income (the $14,000 SE partnership amount is part of it, not extra income), plus the $20,000 distribution, less the $16,800 farm loss, the $3,000 capital-loss limit and the $108 IRA. From there it subtracts three 2026 OBBBA items: the $32,200 MFJ standard deduction, the $13,899 FLSA overtime-premium deduction, and a $510 QBI deduction (20% of the $2,550 net business income). That gives $67,812 of taxable income and $7,641 of tax, less the $2,200 CTC and two $500 other-dependent credits. Most models missed at least one of these: they left out the overtime deduction, applied the pre-TCJA sunset (personal exemptions, $1,000 CTC), counted the $14,000 SE partnership income twice, deducted the repealed DPAD or employer insurance premiums, or added an early-withdrawal penalty or SE tax that is not part of this output." us,scenario_064,federal_refundable_credits,6,llm_error,thresholds_rates,False,,"The refundable child credit is a residual, not an automatic payment: the $2,000-per-child CTC first offsets federal income tax, and only the portion left unused is refundable, subject to the $1,700-per-child ceiling and 15% of earned income above $2,500. This MFJ household's ~$98,810 of income (wages $97,295, $20,000 of 401(k) distributions, $19,350 partnership income, against a farm loss and a $3,000-capped capital loss) produces federal income tax before refundable credits far above $2,000, so the single under-17 child's entire CTC is absorbed nonrefundably, and the 18- and 27-year-old dependents generate only the $500 nonrefundable other-dependent credit. EITC is zero because income is multiples of the MFJ one-child phaseout ceiling, and no refundable AOTC (no students), recovery rebate, or refundable payroll tax credit applies, leaving $0. Every wrong model reported either the full CTC or the $1,700 ACTC ceiling as if refundability were automatic rather than conditional on unused credit." us,scenario_064,free_school_meals_eligible,2,llm_error,taxable_income_or_deductions,False,,"The National School Lunch Program income test counts gross income by source — wages plus taxable retirement distributions — with no netting of farm losses, capital losses, or other tax-code offsets, and no deduction for premiums or medical expenses. PolicyEngine's countable income is $97,295 in wages plus $20,000 in taxable 401(k) distributions = $117,295 against an SPM-unit FPG of $38,680, a ratio of 3.03, far above the 130% free line ($50,284) and the 185% reduced-price line ($71,558). Wisconsin has no universal free meals and no SNAP/TANF categorical eligibility exists here, so the household lands in the PAID tier. The trap is importing tax-return netting (the -$16,800 farm loss and -$18,235 long-term capital loss) into a gross-income benefit test." us,scenario_064,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -536,7 +536,7 @@ us,scenario_064,spouse_medicaid_eligible,1,parse_contract_failure,missing_output us,scenario_064,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,ssi,19,prompt_ambiguity,age_disability,False,,"Every substantive wrong answer assumes that Dependent 1's generic ""is disabled"" flag satisfies SSI's aged/blind/disabled requirement. In the engine, is_ssi_aged_blind_disabled is False for all five members: Head is 48, Spouse 47, Dependent 1 27, Dependent 2 18 and Child 1 12. Because no member passes that test, SSI is $0 before any income, resource, deeming or living-arrangement test applies. The models' later steps (the full federal benefit rate, the one-third or PMV reduction, the Wisconsin supplement, the parental-deeming analysis) all compute a benefit for a person the engine does not count as SSI-disabled. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of the $11,928 federal benefit (five exactly); deepseek-v4-pro, deepseek-v4-pro-0813 and inkling also applied a one-third in-kind reduction the facts do not support." -us,scenario_064,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Wisconsin AGI equals the federal AGI of $114,421.41. That figure is wages minus the $2,315 traditional 401(k) deferral, plus the $20,000 distribution and $19,350 partnership income, minus the $16,800 farm loss, minus $3,000 of capital loss (Wisconsin also allows $3,000 on a joint return), minus the $108 IRA deduction. There is no SE-tax deduction because the farm loss more than offsets the $14,000 SE partnership income. Subtracting the $8,953.27 sliding-scale standard deduction and five $700 exemptions leaves $101,968.14. That amount is taxed at 3.5%/4.4%/5.3% under the 2026 schedule, where the 4.4% bracket runs to roughly $69,000 for joint filers, and no nonrefundable credits apply. The models went wrong in five main ways: an outdated $500 capital-loss cap, the pre-2025 brackets where 5.3% starts near $38,000, a mis-sized sliding standard deduction, missing or misapplied exemptions, and made-up nonrefundable credits." +us,scenario_064,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The correct path starts from Wisconsin AGI equal to federal AGI of $114,421.41. That is wages after the $2,315 401(k) deferral, plus the $20,000 401(k) distribution and $19,350 partnership income. The $14,000 self-employment partnership income is part of the $19,350, not added on top. From that come the $16,800 farm loss, a $3,000 capital loss and the $108 IRA deduction. There is no self-employment tax deduction, because the farm loss wipes out net self-employment earnings. Next it subtracts the $8,953.27 sliding-scale standard deduction and five $700 exemptions, giving taxable income of $101,968.14. That is taxed under the 2026 joint schedule, whose 4.4% bracket runs to roughly $69,000, for $4,599.62, with no nonrefundable credits. Most models went wrong in one of these ways: they added back $2,500 under a $500 capital-loss cap, double counted the self-employment partnership income, used the wrong standard deduction or number of exemptions, used old bracket widths or rates, or invented nonrefundable credits." us,scenario_064,state_refundable_credits,4,llm_error,state_local_rule,False,,"Wisconsin's refundable individual income tax credits are the Homestead Credit, the Wisconsin EITC, the Farmland Preservation Credit, and the Veterans and Surviving Spouses Property Tax Credit; this household qualifies for none, so wi_refundable_credits and therefore state_refundable_credits are $0. The WI EITC is a flat percentage of the federal EITC (4%/11%/34% by number of qualifying children), and the federal EITC is $0 at $97,295 of wages plus partnership and 401(k) distribution income, so the WI EITC is $0. The Homestead Credit is barred both by its household-income ceiling near $24,680 and by the absence of any rent or property tax paid (only a mortgage balance is listed), and the Farmland Preservation Credit requires a farmland preservation agreement or certified zoning, which is unlisted and therefore false. The trap is arriving at an exact $0 rather than assuming that a household with a farm loss and Wisconsin residency picks up some small residual refundable credit." us,scenario_064,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_066,federal_refundable_credits,10,llm_error,credit_phaseout,False,,"The childless EITC has no minimum earned-income floor. It phases in at 7.65% from the first dollar of earnings, and a 40-year-old single filer falls inside the 25–64 age window. With $520 of wages and no investment income, the credit is 0.0765 × $520 = $39.78, and the other refundable credits are all zero. Nine models wrongly assumed the EITC needs a minimum or ""substantial"" earned income, or that it depends on tax liability, so they answered $0. One model got $39.78 and then replaced it with a made-up $234 from a table lookup." @@ -544,27 +544,27 @@ us,scenario_066,head_medicaid_eligible,10,llm_error,categorical_eligibility,Fals us,scenario_066,snap,32,llm_error,thresholds_rates,False,,"This one-person Virginia household is categorically eligible for SNAP through Virginia's broad-based categorical eligibility, which uses the TANF non-cash tests. That test does not disqualify $14,000 in bank assets, so the federal $3,000 resource limit never applies. Earned income is $43.33 a month, so after the standard and shelter deductions net income is $0 and the household receives the full one-person maximum allotment of $298 a month (the FY2026 contiguous-US figure), or $3,576 a year. Models failed in two main ways. Some applied the federal asset limit and gave $0. Others used a stale or guessed maximum allotment, such as $292 (the FY2025 figure), $291, $294, $297 or $300, instead of $298. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 3,596.04 is regenerated as 3,576.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14))." us,scenario_066,state_refundable_credits,35,llm_error,categorical_eligibility,False,,"With $520 of wages and no children, the head gets a federal EITC in the phase-in range: 7.65% × $520 = $39.78. Virginia's refundable EITC pays a share of the federal EITC whether or not the filer owes any Virginia tax, and since tax year 2024 that share is 20%, not the older 15%. That gives 20% × $39.78 = $7.96. Most models wrongly said Virginia has no refundable EITC, that it had expired, or that the federal EITC was zero. The models that did find the credit used the outdated 15% rate and got $5.97–$6." us,scenario_067,dependent1_medicaid_eligible,20,prompt_ambiguity,household_unit_or_filing_status,False,,"PolicyEngine puts dependent1, age 23, in Indiana's ACA adult expansion group with a MAGI income level of 0.00×FPL. The parents' $82,360 in wages and $4,942 in pension never enter dependent1's Medicaid income measure, and dependent1's own $10,800 in disability benefits are not MAGI income. So dependent1 is far below the 138% FPL limit and eligible. All 20 models fell into the same trap in some form. Most added the parents' roughly $87K–$98K to a 3-person ""household MAGI"" of about 317% FPL. A few applied SSI-style or parental income and asset tests that the ACA adult category does not use, or counted the disability benefits as income that fails the limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." -us,scenario_067,dependent1_medicare_eligible,10,llm_error,categorical_eligibility,False,,"Before age 65, Medicare's disability pathway requires 24 months of entitlement to Social Security Disability Insurance, or end-stage renal disease or ALS. Being disabled is not enough on its own. In this household, Dependent 1 has generic ""disability benefits,"" which is a separate income input from Social Security disability. The months of SSDI receipt are unlisted, so they count as 0, and the prompt says not to infer unlisted benefit receipt. Dependent 1 is 23 and has no ESRD, so the person is not Medicare-eligible. Every model treated the disabled flag plus the generic disability income as SSDI entitlement with the waiting period already met." -us,scenario_067,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference uses OBBBA's permanent TCJA structure for 2026: a $32,200 MFJ standard deduction, a 10% bracket up to $24,800, then 12%, and no personal exemptions. AGI is $87,302 because the dependent's $10,800 disability benefits are excluded and no auto-loan-interest deduction applies, so taxable income is $55,102 and tax is $6,116.24. The $500 nonrefundable credit for other dependents applies to the permanently disabled 23-year-old, who counts as a qualifying child because disability waives the age test, and that brings the result to $5,616.24. The wrong answers mostly came from one of these: applying the TCJA-sunset regime that OBBBA eliminated, using stale 2025 standard-deduction and bracket figures, deducting the listed auto loan interest, putting the dependent's disability benefits into AGI, imputing mortgage interest, or getting the $500 dependent credit wrong (leaving it out or swapping in a $2,000+ CTC)." +us,scenario_067,dependent1_medicare_eligible,11,llm_error,age_disability,False,,"Under age 65, PolicyEngine grants Medicare eligibility only for end-stage renal disease or for at least 24 months of Social Security Disability Insurance receipt. The `disability_benefits` input here is a generic disability-benefit income amount, not SSDI. ESRD status and months of SSDI receipt are not listed, so the prompt's default rule sets them to false and 0. A 23-year-old with a disabled flag and $10,800 of generic disability benefits therefore fails every Medicare pathway. Every model treated ""disabled + disability benefits"" as enough for disability-based Medicare, either equating the benefits with SSDI or assuming the 24-month waiting period was already met." +us,scenario_067,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"The correct path uses 2026 law after OBBBA made the TCJA structure permanent. AGI is $87,302 (wages plus taxable pension; the dependent's $10,800 disability benefits are left out), the MFJ standard deduction is $32,200, and taxable income is $55,102. Tax is 10% on the first $24,800 plus 12% on the remaining $30,302, or $6,116.24. Subtracting the $500 nonrefundable credit for other dependents for the 23-year-old disabled dependent gives $5,616.24. Most models missed at least one step: some applied the pre-TCJA rules (personal exemptions, 15% bracket), some used a 2025 or estimated standard deduction or bracket, some took the auto-loan-interest deduction even though no qualifying-vehicle facts were given, some added the dependent's disability benefits to taxable income, and some dropped or inflated the dependent credit." us,scenario_067,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"The 23-year-old disabled household member does not generate a refundable Child Tax Credit because the CTC requires a qualifying child under age 17. With no CTC-qualifying child and no eligibility for the other refundable credit components, federal refundable credits equal $0." us,scenario_067,head_medicare_eligible,1,llm_error,age_disability,False,,"Ordinary age-based Medicare eligibility begins at age 65, and the 60-year-old head is below that threshold. No qualifying disability-based Medicare pathway or other exception is supplied, so the head is not Medicare eligible." -us,scenario_067,local_income_tax,47,prompt_ambiguity,state_local_rule,False,,"The prompt defined local_income_tax by listing only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax. PolicyEngine's local_income_tax also includes Indiana county income tax. Because no county was given, the engine used its default, Adams County, at 1.6% of Indiana AGI: $87,302 federal AGI minus $3,000 of exemptions for three people is $84,302, and $84,302 × 1.6% = $1,348.83. The prompt supports two readings: the list is exhaustive, so an Indiana household owes $0, or it gives examples of every local income tax the engine models, so Indiana county tax counts. Almost every model took the exhaustive reading and answered $0. Three models wrongly said Indiana has no local income tax at all, and one model recognized the county tax but used the wrong amount. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))." +us,scenario_067,local_income_tax,48,prompt_ambiguity,state_local_rule,False,,"The prompt defines local_income_tax by listing only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax. It also gives no Indiana county. Read as a closed list, that wording excludes Indiana's county income tax. PolicyEngine's local_income_tax, however, includes the Indiana county tax, and with no county given it uses Adams County at 1.6%: Indiana AGI is $87,302 minus $3,000 of exemptions for three people, or $84,302, and 1.6% of that is $1,348.83. Almost every model took the closed-list reading and answered $0. Three of them also wrongly said Indiana has no local income tax at all. One model applied a county tax but at the wrong rate. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))." us,scenario_067,payroll_tax,6,llm_error,payroll_tax_base,False,,"The head is the only person with wages. Their $82,360 is taxed at 6.2% for Social Security ($5,106.32) and 1.45% for Medicare ($1,194.22), for $6,300.54 total. Those wages are below the 2026 Social Security wage base and the $250,000 Additional Medicare Tax threshold, and Indiana has no employee state payroll tax. The spouse has no listed wages, so the spouse's wages are $0, and neither the pension nor the disability benefits are FICA wages. The models went wrong by adding wages for the spouse, putting non-wage income into the tax base, or making rounding and arithmetic errors in the simple 7.65% calculation." us,scenario_067,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_067,ssi,17,prompt_ambiguity,age_disability,False,"The question lists Dependent 1 (age 23) as ""is disabled,"" and that person has no earnings, so there is no substantial gainful activity to disqualify them; under 42 U.S.C. §1382c(a)(1)(A) and (a)(3) and 20 CFR 416.202(a), that person is categorically eligible for SSI, yet the engine computes is_ssi_aged_blind_disabled=False and the derivation says no member is flagged disabled in the engine's inputs, so the displayed disability flag is not the input that feeds is_ssi_disabled (an input-mapping mismatch).","The deciding step is SSI's aged/blind/disabled gate. The engine evaluates is_ssi_aged_blind_disabled=False for all three members: the head (60), the spouse (58), and Dependent 1 (23), who has no SSI disability status in the engine's inputs. That makes SSI $0 before any income, resource or deeming test runs. All 17 models treated Dependent 1's listed disability as meeting the SSI disability criterion and subtracted the $900/month disability benefit, less the $20 general exclusion, from the 2026 federal benefit rate (FBR), which lands near $1,368. The outliers come from separate mistakes: an old or invented FBR, or applying the $20 exclusion once a year instead of every month. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_data_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." +us,scenario_067,ssi,18,prompt_ambiguity,age_disability,False,"The question explicitly lists Dependent 1 as ""is disabled"" with no earnings, which satisfies the disability criterion in 42 U.S.C. 1382c(a)(3). The trace still says no member is ""flagged as blind or disabled in the engine's inputs"" and returns is_ssi_aged_blind_disabled=False. This points to the benchmark's disability flag never reaching the engine's SSI disability input (is_ssi_disabled), so SSI fell to $0 instead of ($994 − $880) × 12 = $1,368.","The deciding step is SSI's aged/blind/disabled test. The engine evaluates every member with is_ssi_aged_blind_disabled=False: the head is 60, the spouse is 58, and Dependent 1 is not SSI-disabled under the engine's inputs. So SSI is $0 no matter what the income is. Every model instead took the prompt's 'is disabled' flag as proof that Dependent 1 meets the SSI disability test, then ran the benefit formula against $900/month of disability income. Twelve models arrived at ($994 − $880) × 12 = $1,368. The rest also misstated the federal benefit rate or the general income exclusion. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_data_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." us,scenario_067,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Indiana starts from federal AGI of $87,302 (wages plus taxable pension; the dependent's disability benefits are not in the filers' AGI). It allows only three $1,000 personal exemptions, for the head, spouse and the 23-year-old dependent, totaling $3,000. Indiana has no standard deduction and no extra exemption for a disabled dependent. The $1,500 additional dependent-child exemption applies only to a child under 19 or a full-time student under 24, so this dependent does not get it. That leaves $84,302, taxed at the 2026 statutory flat rate of 2.95%, for $2,486.91. Most models used an outdated or future rate (3.05%, 3.0%, 2.9% or 2.75%), invented a $1,500 disabled-dependent or qualifying-child exemption, or added deductions Indiana does not allow." us,scenario_067,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_068,federal_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"AGI is just the $34,083.53 in gross wages. The $73,000 of educational assistance is excluded from income, and the employer-sponsored insurance premiums are not subtracted from the gross wages. Under 2026 law, which keeps the TCJA structure permanent through OBBBA, the single standard deduction is $16,100. That leaves taxable income of $17,983.53, taxed at 10% up to $12,400 and 12% above it: $1,240 + $670.02 = $1,910.02. No nonrefundable credits apply. The wrong answers come from four main mistakes: adding most or all of the educational assistance to income, taking the ESI premiums out of wages, using pre-TCJA sunset parameters or stale 2024/2025 parameters, and subtracting the refundable EITC from a figure that comes before refundable credits." +us,scenario_068,federal_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"PolicyEngine sets AGI equal to wages of $34,083.53. The $73,000 of educational assistance is not included in income, and the ESI premiums are not subtracted from the stated gross wages. It then applies the 2026 OBBBA single standard deduction of $16,100, which leaves $17,983.53 of taxable income. Under the 2026 brackets, that is 10% on the first $12,400 plus 12% on the remaining $5,583.53, or $1,910.02, with no nonrefundable credits. Most models missed for one of three reasons: they added the educational assistance above the §127 $5,250 cap to wages; they applied pre-TCJA sunset rules (personal exemption and 15% bracket) or 2024/2025 parameters; or they netted the EITC or made-up credits against the tax." us,scenario_068,federal_refundable_credits,4,llm_error,credit_phaseout,False,,"The decisive rule is the 2026 EITC income limit for a single filer with no qualifying children. At $34,084 of wages, the head is above the childless EITC phaseout endpoint, and the household has no qualifying child or other facts generating any refundable federal credit, so every component is zero." us,scenario_068,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Maryland Medicaid eligibility for this 31-year-old requires qualification through a specific MAGI or non-MAGI category. The head's MAGI income is 2.14 times FPL, above the applicable adult MAGI limit, and the head qualifies through no non-MAGI pathway, so the eligibility output is 0." us,scenario_068,payroll_tax,14,llm_error,payroll_tax_base,False,,"PolicyEngine's payroll_tax here is simply the employee FICA pair on the head's $34,084 of employment income: 6.2% Social Security ($2,113.18) plus 1.45% Medicare ($494.21), with the small cent-level offset from the engine's float32 wage representation. The trap has three parts: the FICA wage base is gross wages with no reduction for the $8,389 employer-sponsored insurance premium (PolicyEngine applies payroll tax to employment_income, not to a section 125 net-of-premium wage), the $73,000 educational assistance is student financial assistance rather than employment compensation and never enters the FICA base, and Maryland imposes no mandatory employee payroll contribution in 2026 — the FAMLI employee contribution is not in effect and is not part of this output. Every wrong answer comes from adding one of those three phantom adjustments, or from arithmetic that contradicts the model's own stated components." -us,scenario_068,state_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"Maryland AGI is just the $34,083.53 of wages. The $73,000 of educational assistance stays out of income, and the employer-sponsored insurance premiums do not reduce wages. Starting in 2025, Maryland replaced the old standard deduction (15% of income with a minimum and maximum) with a flat $3,350 for single filers, and the full $3,200 personal exemption applies. That gives $27,533.53 of taxable income: $90 on the first $3,000 plus 4.75% of the rest, or $1,255.34. Most models used the old capped deduction of $2,350 to $2,900, added $67,750 to $73,000 of educational assistance to income, subtracted the insurance premiums, or claimed deductions wiped out all the tax." +us,scenario_068,state_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"Maryland's 2025 reform replaced the old 15%-of-AGI standard deduction, with its roughly $1,800–$2,700 cap, with a flat $3,350 deduction for single filers. Maryland AGI here is just the wages ($34,083.53). It is not reduced by employer-sponsored insurance premiums, and none of the $73,000 educational assistance is added. That gives taxable income of $34,083.53 − $3,350 − $3,200 = $27,533.53 and tax of $90 + 4.75% × $24,533.53 = $1,255.34. Most models still used the old capped deduction (about $2,350–$2,900) and landed at about $1,276–$1,303. Others put some or all of the educational assistance into AGI ($4,100–$5,600), subtracted the ESI premiums (about $895–$900), or claimed the tax was zero." us,scenario_068,state_refundable_credits,2,llm_error,credit_phaseout,False,,"Maryland’s refundable EITC is tied to federal EITC eligibility, and a single 31-year-old filer with no qualifying children and $34,084 of wages is above the childless EITC phaseout ceiling. The federal EITC is therefore zero, producing no Maryland refundable EITC or other applicable refundable Maryland credit." -us,scenario_070,federal_income_tax_before_refundable_credits,38,llm_error,credit_phaseout,False,,"The correct derivation has two steps. First, under the OBBBA-permanent TCJA structure for 2026, AGI of $36,269.67 minus the $16,100 single standard deduction gives taxable income of $20,169.67. Tax is 10% up to $12,400 and 12% above that, which comes to $2,172.36. Second, the nonrefundable Saver's Credit applies at the 10% tier for single filers at this AGI. It covers all $1,924 of elective deferrals and IRA contributions, both traditional and Roth, so it equals $192.40 and leaves $1,979.96. Models went wrong in three main ways: they left out the Saver's Credit, they assumed the TCJA expired (bringing back the personal exemption and the 15% bracket) or used stale 2024/2025 parameters, or they misapplied the credit's rate tier or contribution base." +us,scenario_070,federal_income_tax_before_refundable_credits,39,llm_error,credit_phaseout,False,,"To get the reference, start with AGI of $36,270: $38,000 wages minus the $1,485 traditional 401(k), plus $100 interest, minus $69 traditional IRA and $276 student loan interest. Subtract the 2026 single standard deduction of $16,100 (post-OBBBA, with no personal exemption) to get taxable income of about $20,170. Tax at 10% up to $12,400 and 12% above gives $2,172.36. Then subtract a 10%-rate Saver's Credit of $192.40, figured on $1,924 of contributions that include the Roth 401(k) and Roth IRA, to reach $1,979.96. The wrong answers split into three groups: models that left out the Saver's Credit, models that got its rate or contribution base wrong, and models that used the wrong 2026 parameters (stale or projected standard deduction and brackets, or TCJA-sunset rules with personal exemptions and a 15% bracket)." us,scenario_070,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"The refundable Child Tax Credit requires at least one qualifying child; this household contains only a single 57-year-old adult. With no qualifying child and income above the childless EITC phaseout range, every refundable federal credit component is zero." us,scenario_070,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Illinois’s ACA adult Medicaid pathway requires income no greater than 138% FPL, while the head’s MAGI income is 2.27 times FPL. The head also qualifies through no non-MAGI pathway, so the Medicaid category is NONE and eligibility is No." us,scenario_070,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $38,000 of wages because the prompt lists $300 of health insurance premiums but never identifies them as a pre-tax cafeteria-plan deduction from FICA wages. The correct computation is $38,000 × 6.2% = $2,356 plus $38,000 × 1.45% = $551, totaling $2,907; the Additional Medicare Tax and Illinois employee payroll taxes are zero." -us,scenario_070,state_income_tax_before_refundable_credits,36,llm_error,thresholds_rates,False,,"Illinois tax here is 4.95% × (federal AGI − one personal exemption). Federal AGI is about $36,270: $38,000 wages less the $1,485 traditional 401(k), plus $100 interest, minus the $69 IRA deduction and $276 student loan interest. The 2026 exemption is $2,925, which gives $33,344.67 × 4.95% = $1,650.56. Most models got the base right but used an exemption from an earlier year ($2,425, $2,775 or $2,850), which pushes the tax a few dollars high. Others invented a standard deduction or credits, dropped the student loan interest adjustment, treated health premiums as pre-tax wage reductions, or zeroed out the liability." +us,scenario_070,state_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"Illinois tax here is the flat 4.95% rate applied to federal AGI (about $36,270: wages less the traditional 401(k), plus interest, less the IRA and student loan interest deductions) minus one inflation-indexed personal exemption. That exemption is $2,925 for tax year 2026. Most models used an older exemption amount: $2,850 (2025), $2,775 (2024), or $2,425/$2,450 (2023 or earlier), so they overstated taxable income by $75 to $500. Other models invented an Illinois standard deduction, a renter's credit, or retirement subtractions. Some treated the exemption as a dollar-for-dollar credit, or changed AGI by subtracting health premiums or leaving out student loan interest." us,scenario_070,state_refundable_credits,1,llm_error,credit_phaseout,False,,"Illinois's Earned Income Credit is calculated as a percentage of the federal EITC, so expanding the eligible age range does not create a credit when the federal EITC amount is zero. At this childless filer's income, the federal EITC has fully phased out, making the Illinois refundable credit zero." us,scenario_071,head_medicaid_eligible,8,llm_error,categorical_eligibility,False,,"Age 76 does not itself establish Medicaid eligibility, and this person qualifies through none of the Medicaid pathways represented by the engine. Their MAGI income is 2.28 times FPL, they receive no SSI, and the engine assigns medicaid_category = NONE; Medicare eligibility, housing assistance, and the mere availability of aged or spend-down programs do not create Medicaid eligibility." us,scenario_071,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to listed wages and other covered earnings, not to work hours by themselves. The household has no listed wage or self-employment income, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll tax bases are all zero, yielding payroll_tax of $0." @@ -575,26 +575,26 @@ us,scenario_072,snap,1,llm_error,categorical_eligibility,False,,"The household's us,scenario_072,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Standard Medicare eligibility applies at age 65. The spouse is age 77 in 2026, so the age-based pathway yields Medicare eligibility without any separately listed enrollment or eligibility flag." us,scenario_072,state_income_tax_before_refundable_credits,17,llm_error,state_local_rule,False,,"The head is 80 and the spouse is 77 in 2026, so they were born in 1946 and 1949. Both fall in Michigan's Tier 2 birth cohort (1946-1952) and are past age 67. That qualifies the couple for the Michigan standard deduction of $40,000 on a joint return, which applies against all income, wages included. Their only Michigan-taxable income is $20,000 of wages plus $166 of interest and dividends; Social Security is subtracted and veterans benefits are exempt. The $40,000 deduction wipes out that $20,166 entirely, so tax before refundable credits is $0. Every wrong model left a positive base: most subtracted only the ordinary personal exemptions, one used the wrong birth cohort, and one found the $40,000 deduction but shrank it by Social Security benefits." us,scenario_073,head_medicaid_eligible,8,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI (42 CFR 435.603(e), which follows IRC 36B(d)(2)(B)) adds the untaxed portion of Social Security benefits back into income. That means the full $22,646 of SSDI counts, on top of the $3,108 pension. The head has no wages, so the traditional 401(k) and IRA contributions reduce nothing. MAGI is about $25,754, or 1.60 times FPL, which is above Michigan's 138% FPL limit for the adult expansion group. No other category applies: the head is 57, gets no SSI, is not pregnant, has no dependent children and has no disability flag, so the engine assigns medicaid_category NONE. Six models wrongly left untaxed SSDI out of MAGI or simply called the income 'low'. One subtracted 401(k) deferrals the head cannot make without wages. One treated SSDI receipt on its own as a disability route into Medicaid." -us,scenario_073,head_medicare_eligible,18,prompt_ambiguity,age_disability,False,,"The head is 57, so the age-65 route is out. The only other route is disability-based Medicare, which starts only after 24 months of Social Security disability benefits (42 U.S.C. 426(b)). The household lists this year's SSDI amount but not how long the head has received it. The prompt sets every unlisted number to 0 and every unlisted status to false, so the months of SSDI receipt count as 0 and the waiting period is not met. That leaves the head not Medicare eligible. All 18 models made the same mistake: they treated this year's SSDI as proof of disability-based Medicare, either by skipping the 24-month rule or by assuming it was already met. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_073,head_medicare_eligible,19,prompt_ambiguity,age_disability,False,,"The head is 57, so they miss the age-65 Medicare test. Under 65, Medicare eligibility requires at least 24 months of SSDI receipt (or ESRD). In PolicyEngine this is a separate months-of-SSDI-receipt input, and here it is unlisted, so it is 0 under the prompt's rule that unlisted numeric inputs are zero and unlisted status facts are false. Every model read the $22,646 SSDI amount as proof that the 24-month waiting period was met. Even the full-year constant-status framing only shows 12 months of receipt, which is under 24, and no ESRD is listed, so the head is not Medicare eligible. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_073,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earnings, not Social Security disability benefits or taxable private pension income. With no wage, salary, or self-employment income listed, the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components are all zero." -us,scenario_073,snap,44,llm_error,categorical_eligibility,False,,"The household has one disabled member (SSDI). Its monthly gross income is $2,146, about 165% of FPL, which is under Michigan's 200% FPL limit for broad-based categorical eligibility, so the household is categorically eligible through TANF non-cash benefits. That means the 100% FPL net income test does not apply. Net income is $1,755, so the formula benefit is $298 − 0.3 × $1,755, which is below zero. Categorically eligible one- and two-person households still get the minimum allotment, which is $24 a month (about 8% of the $298 one-person maximum), or $288 a year. Most models applied a net income test, a 130% gross test, or an asset limit and stopped at $0, or they never applied the minimum allotment. A few applied the minimum but used the outdated $23 or a wrong $25 amount." +us,scenario_073,snap,45,llm_error,categorical_eligibility,False,,"Michigan's broad-based categorical eligibility (TANF non-cash) makes this one-person household categorically eligible because its $2,146 monthly gross income is below 200% FPL. Categorically eligible households skip the net income test, and the resource test does not bind. Net income is $1,755, so the 30% expected contribution ($527) exceeds the $298 maximum allotment. Because categorically eligible one- and two-person households get the minimum allotment, the benefit is $24 a month, or $288 a year. Most models either ran the 100% FPL net income test, or did not know the minimum allotment and returned $0. A few used the old $23 or a wrong $25 minimum, or overstated medical deductions." us,scenario_074,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_074,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_074,head_medicaid_eligible,6,llm_error,health_coverage,False,,"Medicaid MAGI is not taxable income: 26 U.S.C. §36B(d)(2)(B) adds back to AGI all Social Security benefits excluded from gross income, so the head's $33,640 SSDI counts in full alongside the $384 taxable IRA distribution, producing 2.13 x FPL — roughly 213% of poverty against Louisiana's 138% FPL expansion-adult ceiling. Every wrong model shrank that base: four dropped SSDI entirely as ""non-taxable,"" one halved it using the §86 provisional-income 50% factor, and one asserted a disability pathway. The non-MAGI routes are also closed: the SSI-related aged/blind/disabled categories require SSI receipt (SSI = $0) or age 65+ (the head is 54), and the head is not a tax-unit dependent, so medicaid_category resolves to NONE." -us,scenario_074,head_medicare_eligible,28,prompt_ambiguity,age_disability,False,,"The head is 54, so the age-65 test fails. The only other route is the disability route, which requires 24 months of Social Security disability entitlement. PolicyEngine checks a count of months receiving SSDI. It does not check the SSDI dollar amount. The household gives no months of SSDI receipt and no disability status, and the prompt says unlisted numbers are 0 and unlisted statuses are false. So the 24-month requirement is not met and the head is not Medicare eligible. Every model treated a positive SSDI amount as proof that the waiting period was already done, which adds a fact the prompt never gave. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_074,head_medicare_eligible,29,prompt_ambiguity,age_disability,False,,"PolicyEngine makes a person Medicare-eligible in one of three ways: age 65 or older, at least 24 months of receiving Social Security disability benefits (a separate months-of-receipt input), or ESRD. A positive SSDI amount does not satisfy the disability route by itself. The head is 54, and neither SSDI duration nor ESRD is listed. The prompt says to treat unlisted numeric inputs as 0 and unlisted status inputs as false, so the head has 0 months of SSDI receipt and does not meet the 24-month test. Every model either assumed the waiting period was met or treated SSDI income alone as enough, and answered Yes. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_074,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered employment earnings, not to Social Security disability benefits, pension income, IRA distributions, or tax refunds. The household has no listed wages, and the prompt requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and the resulting payroll tax is $0." us,scenario_075,federal_income_tax_before_refundable_credits,28,llm_error,thresholds_rates,False,,"The 2026 tax falls under the OBBBA, which made the TCJA structure permanent. That means a $16,100 single standard deduction, no personal exemption, and single brackets of 10% up to $12,400, 12% up to $50,400 and 22% up to $105,700. Ordinary taxable income of $86,960 is therefore taxed at $13,843, and the $6,226 of long-term gains and qualified dividends stacked on top is taxed at 15% ($934), for a total of $14,777. Nothing else applies: a 62-year-old gets no age-65 additional deduction or senior deduction, disability does not raise the standard deduction, and the Credit for the Elderly or Disabled is fully phased out at $109,286 AGI. The largest group of models assumed the TCJA sunset (personal exemption plus 10/15/25% brackets). Another group used 2025 or undersized 2026 standard deductions and bracket thresholds. A few computed $14,777 correctly but then submitted a different number." us,scenario_075,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_075,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not establish Florida Medicaid eligibility. The head qualifies through none of Florida's Medicaid pathways, so the engine assigns medicaid_category NONE; the 6.85× FPL income level also rules out any applicable MAGI category." -us,scenario_075,head_medicare_eligible,15,llm_error,age_disability,False,,"Under age 65, Medicare disability eligibility depends on 24 months of Social Security Disability Insurance entitlement, or on end-stage renal disease. A general ""is disabled"" flag does not qualify anyone. This head is 62 and has no Social Security disability benefits: there are no months of SSDI receipt and no ESRD. Their income is wages, pensions, dividends and capital gains. So the age test (65+) fails and neither non-age pathway applies. Every model instead treated the disability flag, or age 62, as enough for Medicare." +us,scenario_075,head_medicare_eligible,16,llm_error,age_disability,False,,"Medicare covers people aged 65 or older. Under 42 USC 426(b), people under 65 qualify through disability only after 24 months of entitlement to Social Security disability benefits, and there are separate ESRD and ALS pathways. PolicyEngine follows this rule: the age test fails at 62, and the disability pathway requires 24 or more months of SSDI receipt. This head has no Social Security disability income, since unlisted numeric inputs are 0, and has no ESRD or ALS, so the head is not eligible. Every model treated the generic 'is disabled' flag alone as enough for Medicare and skipped the requirement to have received SSDI for 24 months." us,scenario_075,payroll_tax,5,llm_error,other,False,,"Only the $12,656 in wages is subject to payroll tax. Pension income, dividends and capital gains are not FICA wages. Employee Social Security at 6.2% plus employee Medicare at 1.45% comes to about $968. Wages are far below the Additional Medicare Tax threshold, and Florida has no employee state payroll tax. No model was misled by the tax base. The errors are arithmetic slips, a final value that contradicts the model's own correct working, and missing or unparsed answers." us,scenario_076,child2_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC serves children only through their fifth birthday and generally requires household income at or below 185% of the federal poverty guideline. The model also inverted the income comparison: $173,820 is far above 185% of the poverty guideline for a three-person household, not 184% of the guideline." -us,scenario_076,federal_income_tax_before_refundable_credits,41,llm_error,thresholds_rates,False,,"The correct path uses AGI of $164,220. Wages are not reduced by the ESI premium, and child support is excluded. Subtract the 2026 head-of-household standard deduction of $24,150 and an $800 QBI deduction (20% of the $4,000 rental income) to get taxable income of $139,270. The 2026 head-of-household brackets (10% to $17,700, 12% to $67,450, 22% to $105,700, then 24%) give $24,211.80 of tax. The full $4,400 CTC ($2,200 per child) and the $200 CDCC are nonrefundable against that liability, leaving $19,611.80. The wrong answers came from a few mistakes: assuming the TCJA expired in 2026 (personal exemptions, a $1,000 CTC that phases out); using 2025 brackets or a $2,000-per-child CTC; dropping the QBI deduction; subtracting the ESI premium from wages; estimating mortgage interest that the prompt never listed; treating the refundable ACTC cap as shrinking the nonrefundable CTC; and using the wrong filing status." +us,scenario_076,federal_income_tax_before_refundable_credits,42,llm_error,thresholds_rates,False,,"The correct 2026 calculation applies the permanent OBBBA parameters. AGI is $164,220 (wages, rent and interest; child support is excluded, and the ESI premium is not subtracted from the listed gross wages). Subtracting the $24,150 head-of-household standard deduction and the $800 QBI deduction on the rental income leaves taxable income of $139,270. The 2026 HoH brackets (10% to $17,700, 12% to $67,450, 22% to $105,700, then 24%) give $24,211.80. The full $4,400 CTC ($2,200 × 2) and the $200 CDCC are nonrefundable and are fully absorbed by that tax. Models missed this in several ways: assuming the TCJA sunset (personal exemptions, the $1,000 CTC with its phase-out), using 2025 or made-up brackets and standard deductions, leaving out the QBI deduction, subtracting the ESI premium, choosing the wrong filing status, imputing mortgage interest, or misreading the CTC so that most of it was treated as refundable instead of nonrefundable." us,scenario_076,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit is not an entitlement of $1,700 per child; it is only the portion of the $2,200-per-child CTC that survives after the credit offsets tax liability, and it is further capped at 15% of earned income over $2,500 and at $1,700 per child. This head-of-household filer has AGI of $164,220 (wages $160,000 + rents $4,000 + interest $220; child support is not taxable), below the $200,000 HoH phase-out threshold, so the full $4,400 CTC is available — but taxable income near $140,000 after the 2026 HoH standard deduction and the QBI deduction produces federal tax well above $25,000, so the entire CTC is absorbed nonrefundably and nothing spills into refundable_ctc. EITC is zero on this income, and there is no AOTC, recovery rebate, or refundable payroll tax credit here, so federal refundable credits are $0. Every wrong model computed the ACTC cap and then reported the cap itself as a payment, skipping the tax-liability offset that defines the refundable portion." us,scenario_076,payroll_tax,12,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are calculated on the full $160,000 of wages in the computation trace; the listed $6,480 employer-sponsored insurance premium is not subtracted from the payroll-tax base. This yields $9,920 of Social Security tax plus $2,320 of Medicare tax, for $12,240 total, with no Additional Medicare Tax or mandatory Idaho employee payroll tax." us,scenario_076,self_employment_tax,2,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to net earnings from a trade or business, and the prompt lists no self-employment income. The $4,000 of rental income remains rental income and does not create a self-employment tax base, so the computation yields $0." -us,scenario_076,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"Idaho taxable income is $164,220 AGI minus the $24,150 HoH standard deduction, the $1,000 Idaho child-care subtraction and the $800 QBI deduction, which leaves $138,270. The 2026 flat 5.3% rate applies only above the indexed head-of-household zero-rate band of about $9,622, which gives $6,818.34. No Idaho nonrefundable credit is applied, and the $10 permanent building fund tax is then added. Most models taxed the whole amount at 5.3%, or at the outdated 5.695%/5.8% rates. Many also dropped the child-care subtraction or the QBI deduction, subtracted a $410 Idaho child tax credit that is not part of the 2026 computation, and left out the $10 permanent building fund tax. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount. Reference affirmed (Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601)." -us,scenario_076,state_refundable_credits,40,llm_error,thresholds_rates,False,,"Idaho's state refundable credit here is the grocery credit. It has no income phase-out, is refundable, and is allowed for each personal exemption: the head and both children. For 2025 and later it is $155 per person under 65, so this household gets 3 × $155 = $465. The wrong answers fall into two groups. Models answering $360 or $300 used the old $120 or even older $100 per-person amounts. Models answering $0 wrongly treated the grocery credit as nonrefundable or income-limited, or missed it entirely." +us,scenario_076,state_income_tax_before_refundable_credits,48,llm_error,state_local_rule,False,,"Idaho's 2026 tax starts from federal AGI of $164,220. Subtracting the $24,150 HoH standard deduction, the $1,000 Idaho dependent-care subtraction and the $800 QBI deduction (20% of the $4,000 rental income) leaves $138,270 of Idaho taxable income. Head-of-household filers use the joint rate schedule, so the 5.3% flat rate applies only above the $9,622 zero-rate amount: 5.3% × $128,648 = $6,818.34. No nonrefundable credit reduces that figure, and the $10 permanent building fund tax is added on top for $6,828.34. Most models skipped one or more of these steps: they taxed from the first dollar or used the single $4.8k–5k threshold, used the old 5.695%/5.8% rates, left out the dependent-care and QBI subtractions, subtracted a $410 child credit, or forgot the $10 building fund tax. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount. Reference affirmed (Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601)." +us,scenario_076,state_refundable_credits,41,llm_error,thresholds_rates,False,,"Idaho's grocery credit is refundable, has no income phase-out for a household like this one, and is paid per person for the filer and each dependent. In 2026 it is $155 for each member under 65, so this three-person household gets 3 × $155 = $465. Models that gave $0 wrongly treated the credit as nonrefundable or limited by income. The $360, $300 and $510 answers used old or invented per-person amounts ($120, $100, $170) instead of the $155 that applies after the 2025 increase. The $410 answers used Idaho's child tax credit, which is nonrefundable, in place of the grocery credit." us,scenario_077,federal_income_tax_before_refundable_credits,30,llm_error,thresholds_rates,False,,"The correct path is short. AGI is $26,006 in wages minus the $3,000 cap on net capital losses, which gives $23,006. Under OBBBA, the TCJA structure is permanent, so the 2026 single standard deduction is $16,100 with no personal exemption. That leaves $6,906 of taxable income, all taxed at 10%, for $690.62. No nonrefundable credits apply. Most models went wrong in one of three ways: they used a stale or guessed standard deduction ($15,000, $15,350, $15,750, and so on), they applied TCJA-sunset rules ($8,300 standard deduction plus a $5,300 exemption), or they subtracted the ESI premiums from wages that were already stated as gross. Several others computed about $691 and then submitted a different number or zeroed out the tax for no reason." us,scenario_077,federal_refundable_credits,13,llm_error,credit_phaseout,False,,"The only potentially applicable refundable credit is the childless EITC, and its phaseout uses the greater of earned income or adjusted gross income. The $26,006 of wages is earned income; employer-sponsored insurance premiums do not reduce that amount, and at this earnings level the childless EITC is fully phased out. With no qualifying child or student and no applicable rebate or payroll credit, every refundable component is zero." us,scenario_077,head_medicaid_eligible,10,llm_error,taxable_income_or_deductions,False,,"Louisiana's expansion pathway requires MAGI at or below the applicable adult limit, but the engine computes the head's MAGI income level as 1.44 times FPL, so that pathway fails. The listed employer-sponsored insurance premium is not established as a pre-tax payroll deduction and cannot simply be subtracted from wages; after the applicable MAGI computation, the head qualifies through no Medicaid category." @@ -602,23 +602,23 @@ us,scenario_077,head_wic_eligible,1,parse_contract_failure,missing_output,False, us,scenario_077,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are computed from the applicable payroll-tax wage base, and the reference calculation applies the 6.2% and 1.45% employee rates to the full $26,006 of wages. The employer-sponsored insurance premium input does not reduce that wage base in this calculation, producing $1,612.38 of Social Security tax plus $377.09 of Medicare tax, or $1,989.47." us,scenario_077,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_077,snap,5,llm_error,categorical_eligibility,False,,"Louisiana’s broad-based categorical eligibility gross-income screen does not itself guarantee a SNAP allotment; the household must still produce an eligible benefit under the net-income and allotment calculations. SNAP countable income does not subtract capital losses, a mortgage balance is not a shelter expense, and a non-elderly, nondisabled household cannot deduct medical expenses. With no listed deductible shelter payment and countable wages of $26,006, this one-person household receives $0." -us,scenario_077,state_income_tax_before_refundable_credits,47,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2025 reform (Act 11 of the 2024 special session) set a flat 3% rate and a $12,500 single standard deduction, and indexed that deduction for inflation starting in tax year 2026, which makes it $12,835. Federal AGI is $23,006 because net capital losses are capped at $3,000. Subtracting $12,835 leaves $10,171.18 of taxable income, and 3% of that is $305.14. The most common wrong answer, $315.18, came from using the unindexed 2025 deduction of $12,500. Most other misses applied repealed pre-2025 Louisiana law (1.85%/3.5% brackets, a $4,500 personal exemption, the deduction for federal income tax paid), took ESI premiums out of gross wages a second time, or said Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)." +us,scenario_077,state_income_tax_before_refundable_credits,48,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2025 reform set a flat 3% rate and a $12,500 single standard deduction, indexed from 2026 under R.S. 47:294(B). PolicyEngine computed $12,835 from published CPI-U, giving the frozen reference of 3% × ($23,006.18 − $12,835) = $305.14. Louisiana first published its 2026 return deduction, $12,838, in RIB 26-019 on September 28, 2026; that gives $305.05, and this output is excluded from scoring under d994. Of the 48 models, 20 answered $315.18 or $315, amounts consistent with the $12,500 deduction and flat 3% rate; three more initially reached about $315 that way before raising their submitted answers to $446–$496. Other models used repealed graduated brackets or personal exemptions, deducted federal income tax despite that deduction ending after 2021, subtracted ESI premiums from wages, ignored the $3,000 capital-loss cap, or claimed Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)." us,scenario_077,state_refundable_credits,4,llm_error,state_local_rule,False,,"The models applied Louisiana’s former refundable earned income tax credit, equal to 5% of the federal EITC, to tax year 2026. Louisiana no longer provides that refundable credit in 2026, so the Louisiana refundable-credits computation returns zero regardless of an estimated federal EITC amount." us,scenario_077,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_078,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"Under OBBBA rules for 2026, TCJA brackets and the $16,100 standard deduction stay in place, and the SALT cap rises to $40,400. There is no phase-down here because MAGI of about $200,137 is far below $505,000. Real estate taxes of $26,460 plus the Maryland state and county income tax on about $200k of wages exceed $40,400, so SALT is capped at $40,400. With $28,210 of mortgage interest, itemized deductions total $68,610 and taxable income is $131,526.91, which gives $24,164.46 at the 10/12/22/24% brackets (24% bracket starting at $105,700). The models went wrong in one of four ways: applying the old $10,000 SALT cap, assuming the TCJA sunset (personal exemptions and the 15/25/28% brackets), leaving Maryland income tax out of SALT entirely, or counting too little Maryland state-plus-county income tax so SALT stayed under the cap. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." +us,scenario_078,federal_income_tax_before_refundable_credits,48,prompt_ambiguity,taxable_income_or_deductions,False,,"Under OBBBA, the 2026 SALT cap is $40,400. MAGI here is about $200k, well below the $505k phase-down threshold. The SALT deduction counts the $26,460 of real estate tax plus both Maryland state income tax and Maryland county (local) income tax. Together these exceed $40,400, so SALT is capped at $40,400. Adding $28,210 of mortgage interest gives $68,610 in itemized deductions and $131,526.91 of taxable income, which produces $24,164.46 of tax under the OBBBA-era 2026 single brackets (12,400/50,400/105,700). The models went wrong in three main ways: they used the old $10,000 TCJA cap, they left state and local income taxes out of SALT (or counted only the state portion), or they assumed TCJA had expired, bringing back personal exemptions and pre-2018 brackets. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." us,scenario_078,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_078,local_income_tax,1,llm_error,state_local_rule,False,,"The requested local_income_tax output is limited to the separately modeled local taxes listed in the prompt: NYC, Philadelphia, Kansas City, and St. Louis. Maryland county income tax is not included in this output, so a Maryland household outside those modeled jurisdictions has a value of zero." us,scenario_078,payroll_tax,27,llm_error,thresholds_rates,False,,"The correct payroll tax uses the 2026 Social Security wage base of $184,500: 6.2% × $184,500 = $11,439, plus 1.45% Medicare on all $198,505 of wages ($2,878.32), for $14,317.32. Wages are below the $200,000 Additional Medicare Tax threshold, and Maryland charges no mandatory employee payroll tax in 2026. Most wrong answers used an outdated or wrongly projected wage base. Others added a Maryland FAMLI contribution, which is not collected in 2026, or invented a Maryland payroll tax. Several models worked out exactly $14,317.32 and then submitted a different number." -us,scenario_078,state_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"Maryland AGI is federal AGI of $200,136.91 minus the $1,631.91 state tax refund subtraction, which leaves $198,505. Maryland itemized deductions are federal itemized deductions minus state income taxes. Here that means mortgage interest of $28,210 plus the full $26,460 in real estate taxes, since that amount is under the 2026 federal SALT cap of $40,400. This $54,670 total is reduced by 7.5% of federal AGI above $200,000 ($10.27), giving $54,659.73. A single filer above $150,000 gets no personal exemption, so taxable income is $143,845.27. Maryland brackets (2/3/4/4.75/5/5.25%) then give $6,936.88. Most wrong models did one of four things: took the small Maryland standard deduction instead of itemizing, capped property taxes at $10,000, left the refund in Maryland AGI or claimed an exemption, or added county income tax." +us,scenario_078,state_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"The correct 2026 Maryland calculation has five steps. First, subtract the $1,632 state tax refund from federal AGI to get MD AGI of $198,505. Second, itemize $28,210 of mortgage interest and $26,460 of real estate taxes; the property taxes sit under the OBBBA SALT cap, not a $10,000 cap. Third, reduce that total by 7.5% of federal AGI over $200,000 ($10.27). Fourth, apply a $0 personal exemption, because single filers above $150,000 get none. Fifth, run the 2%–5.25% brackets on $143,845.27. Most wrong answers either used the small MD standard deduction instead of itemizing, kept the $1,632 refund in the base, capped property taxes at $10,000, kept a partial exemption, used the wrong itemized phase-out, or added county tax." us,scenario_078,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_079,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The operative rule is categorical, not income-based: the head's blindness, disability, and $7,560 of SSDI leave countable income below the SSI federal benefit rate, producing $5,286 of SSI, and Arizona is a 1634 state where SSI receipt automatically confers Medicaid with no MAGI test. Both models ran an income/coverage screen instead of the SSI-recipient category, and neither recognized that SSI here is a computed output of the listed facts rather than an unlisted input. The screen they ran also points to eligibility rather than away from it — 0.35 x FPL is far under Arizona's 138% FPL expansion threshold — so 'No' is unreachable by either the categorical or the MAGI route." -us,scenario_079,head_medicare_eligible,24,prompt_ambiguity,age_disability,False,,"Medicare eligibility here has two routes: reaching age 65, or completing 24 months of Social Security disability entitlement (42 U.S.C. 426(b)). PolicyEngine checks the second route against a months-of-SSDI-receipt input. The household lists SSDI income but no months of receipt, and the prompt says to treat unlisted numeric inputs as 0 and not to infer unlisted facts, so the waiting period is not met. Even if Head received SSDI for all of 2026, that adds up to only 12 months. With Head aged 58, neither route applies, but all 24 models treated SSDI receipt or the disability flag as enough for Medicare, and several simply assumed the 24-month wait had already passed. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_079,head_medicare_eligible,25,prompt_ambiguity,age_disability,False,,"Under 65, Medicare eligibility through disability requires 24 months of Social Security disability entitlement. ESRD and ALS are the only exceptions, and neither is listed. The household facts give no duration of SSDI receipt, and the prompt says unlisted numeric inputs are 0, so Head has 0 months of receipt and has not met the waiting period. At age 58, Head also fails the age-65 test, so Head is not eligible. Every model treated current SSDI receipt, or disability status alone, as enough for Medicare. Several of them explicitly assumed the 24-month waiting period was already met. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_079,local_income_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_079,snap,47,llm_error,state_local_rule,False,,"Each month the household has $1,511 of countable unearned income: $630 of SSDI plus $881 of SSI for the couple, which the models had to work out themselves. Arizona gives elderly/disabled households with more than $35 a month in medical costs a $145 standard medical deduction instead of their actual excess, which here is only $26. With the $209 standard deduction, net income is $1,157, so the household's expected contribution is $348 and the benefit is $546 − $348 = $198 a month ($2,376 a year). The models split into three groups: about 15 left out the SSI income and landed near $4,500–$5,200; about 15 included SSI but used the $26 actual medical excess and landed near $1,900–$2,100; and the rest made up asset, income-limit, shelter, or SSI-exclusion reasons to answer $0." +us,scenario_079,snap,48,llm_error,state_local_rule,False,,"Getting this right takes two steps. First, the couple qualifies for SSI: both are blind and disabled, and the 2026 couple FBR of about $1,491 minus $610 of countable SSDI ($630 less the $20 disregard) leaves $881/month. That SSI counts as unearned income for SNAP, so gross income is $1,511. Second, because the household's actual medical costs ($61/month) exceed the $35 threshold, Arizona replaces the actual $26 excess with its $145 Standard Medical Deduction. That gives net income of $1,157, a contribution of $348, and $546 − $348 = $198/month ($2,376/year). Most models left out SSI and landed at about $4,900–5,200. The best group included SSI but used the actual $26 excess instead of Arizona's $145 SMD, which gives about $163/month ($1,956). The $0 answers wrongly applied asset, income, shelter-cost, or SSI-exclusion rules." us,scenario_079,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The spouse is blind and disabled with zero countable income and $1,045 in resources, so she receives SSI ($5,286 in the household trace); Arizona is a 1634 state, where SSI receipt confers automatic Medicaid through the SSI-recipient category with no separate income test. Both models evaluated only an income/MAGI-style adult pathway and never checked the SSI-recipient categorical route that the engine actually used (medicaid_category = SSI_RECIPIENT). The error is compounded because even the MAGI route qualifies here: household MAGI is 0.35 × FPL, far under Arizona's 138% FPL expansion-adult ceiling, and Indian Health Service coverage is not minimum essential coverage that bars Medicaid. Zero-income, SSI-receiving disabled adults in an expansion state are Medicaid-eligible on two independent pathways, so ""No"" is unreachable from the stated facts." us,scenario_079,spouse_medicare_eligible,4,llm_error,age_disability,False,,"The spouse is 55, so the only route to Medicare is the disability pathway. That route needs 24 months of Social Security disability (SSDI) entitlement, or ESRD/ALS. Being disabled or blind does not qualify anyone for Medicare by itself. The spouse has no Social Security disability income at all (the $7,560 belongs to the head), and no months of SSDI receipt or ESRD are listed, so all of these inputs default to zero or false. Every model treated ""is disabled"" as enough for Medicare and skipped the SSDI entitlement and 24-month waiting-period requirement." us,scenario_079,ssi,34,llm_error,thresholds_rates,False,,"Both spouses are blind and disabled, so they form an eligible SSI couple. Their countable resources are $1,045 in the bank, because the home and one vehicle are excluded, which is under the $3,000 couple limit. The head's $630/month SSDI minus the $20 general exclusion leaves $610/month of countable income. The 2026 couple federal benefit rate is $1,491/month, so the benefit is $1,491 − $610 = $881/month, or $10,572/year. Most models got the structure right but used a stale or misprojected couple rate ($1,415, $1,450, $1,482–$1,490, or inflated values). The rest wrongly found the couple ineligible, refused to compute SSI, or made arithmetic or submission errors." -us,scenario_079,state_refundable_credits,40,llm_error,categorical_eligibility,False,,"Arizona's Increased Excise Tax Credit (A.R.S. §43-1072.01) is refundable. It pays $25 for each personal and dependent exemption, up to $100, to filers whose federal AGI is at or below $25,000 (MFJ/HOH) or $12,500 (single/MFS). It needs no earned income, tax liability, children, or minimum AZ taxable income. This MFJ couple's only income is non-taxable SSDI, so their federal AGI is $0, they are under the cap, and the credit is 2 × $25 = $50. Almost every model treated zero income or zero liability as ruling out refundable AZ credits. Several also misstated the credit as nonrefundable or tied to EITC, earnings, or filing thresholds." +us,scenario_079,state_refundable_credits,41,llm_error,categorical_eligibility,False,,"Arizona's Increased Excise Tax Credit (A.R.S. 43-1072.01) is refundable. It pays $25 for each taxpayer, spouse and dependent, up to a $100 cap, when federal AGI is no more than $25,000 for joint filers. It does not depend on earned income, children or any tax liability. This couple's only income is $7,560 of SSDI, so their AGI is far below the limit, and the credit is 2 × $25 = $50. Nearly every model either did not know the credit exists, thought it was nonrefundable, or wrongly assumed it requires taxable income, earnings or children. As a result, they answered $0." us,scenario_080,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -630,30 +630,30 @@ us,scenario_080,local_income_tax,1,parse_contract_failure,missing_output,False,, us,scenario_080,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_080,snap,47,reference_engine_defect,categorical_eligibility,False,,"To reach $3,596.04, three steps are needed. First, Pennsylvania's broad-based categorical eligibility (through the TANF non-cash program) waives the SNAP asset test, so the $19,828 bank balance does not matter. Second, the $2,000 energy subsidy triggers the heat-and-eat standard utility allowance of $857 a month. Together with the standard deduction, that allowance brings net income to $0 every month. Third, the benefit is the one-person maximum allotment: $298 a month for January–September (FY2026) and $304.68 a month for October–December (FY2027), not one flat monthly figure times 12. The largest group of wrong models applied the asset test and answered $0. A second group skipped the utility allowance and subtracted 30% of net income. The rest reached the maximum benefit but used the FY2025 allotment ($291–$293) or $298 for all 12 months. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (categorical_eligibility). The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Reference is an engine defect; output excluded from scoring (7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04))." +us,scenario_080,snap,48,reference_engine_defect,categorical_eligibility,False,,"Getting to the reference takes three steps. First, Pennsylvania's broad-based categorical eligibility through its TANF non-cash program waives the SNAP asset test, so the $19,828 bank balance does not disqualify the household. Second, the $2,000 energy subsidy triggers Pennsylvania's $857 standard utility allowance (heat-and-eat). Together with the standard deduction, the excess shelter deduction brings the $301.33 monthly countable income ($300 assistance + $1.33 dividends) down to $0 net income. Third, the household gets the one-person maximum allotment of $298 for January–September and the uprated $304.68 for October–December, which totals $3,596.04. Nineteen models wrongly returned $0, mostly by applying the asset limit. Twelve skipped the utility-allowance-driven shelter deduction and subtracted 30% of net income. Sixteen got to the maximum but used a stale monthly maximum or held it flat all year without the October 2026 increase. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (categorical_eligibility). The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Reference is an engine defect; output excluded from scoring (7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04))." us,scenario_080,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to be aged 65 or older, blind, or disabled under the program's categorical determination. The engine input sets the sole person's SSI aged/blind/disabled status to false, so benefit-rate and countable-income calculations never produce an award." us,scenario_080,state_income_tax_before_refundable_credits,8,llm_error,credit_phaseout,False,,"Pennsylvania first applies its 3.07% rate to the $940 of taxable dividends and gains, then applies the nonrefundable Tax Forgiveness credit in computing income tax before refundable credits. The household's eligibility income remains within the range for 100% forgiveness, so the credit eliminates the preliminary tax and yields $0." us,scenario_080,state_refundable_credits,2,llm_error,state_local_rule,False,,"Pennsylvania Schedule SP tax forgiveness is a nonrefundable credit that reduces Pennsylvania income-tax liability; it is not included in state refundable credits. With no qualifying Pennsylvania refundable credit, the requested aggregate is $0." us,scenario_080,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_081,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The deciding step is the SALT part of itemized deductions. For 2026, OBBBA sets the SALT cap at $40,400, so the filer deducts the full $5,643 property tax plus the Massachusetts income tax. PolicyEngine counts that state tax as estimated withholding of 5% × (AGI $174,088.12 − $4,400) = $8,484.41, not the $8,238.41 final MA liability. This gives itemized deductions of $31,027.41 and, after the $1.24 QBI deduction, taxable income of $143,059.47. OBBBA's permanent 2026 single brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, then 24%) turn that into $26,932.27. Most wrong answers made one of these errors: applied the old $10,000 TCJA cap, left MA income tax out of SALT, assumed TCJA expired (personal exemption and 15/25/28% brackets), took the standard deduction, subtracted ESI premiums from wages, or used the MA liability instead of the withholding estimate (about $60 high). Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)." +us,scenario_081,federal_income_tax_before_refundable_credits,48,prompt_ambiguity,taxable_income_or_deductions,False,,"The answer depends on how the itemized SALT deduction is built. 2026 law keeps TCJA brackets and has no personal exemption. The OBBBA SALT cap is $40,400, so SALT is not limited here. It equals the $5,643 of real estate tax plus PolicyEngine's estimated Massachusetts income tax withholding of 5% × ($174,088.12 − $4,400) = $8,484.41. With $16,900 of mortgage interest, itemized deductions total $31,027.41. Subtracting those and a $1.24 QBI deduction gives taxable income of $143,059.47, and 2026 single brackets produce $26,932.27. Models fell into four groups: some left state income tax out of SALT, some applied the old $10,000 cap, some assumed the TCJA sunset (personal exemption plus 15/25/28% brackets), and some used a slightly different MA tax figure than the $8,484.41 withholding estimate. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)." us,scenario_081,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_081,payroll_tax,38,prompt_ambiguity,state_local_rule,False,,"Federal FICA is the same in every correct path. The $175,002 in wages is below the 2026 Social Security wage base, so the employee owes 6.2% Social Security ($10,850.12) and 1.45% Medicare ($2,537.53), with no Additional Medicare Tax. That totals $13,387.65. The reference adds the Massachusetts PFML employee contribution at the maximum share an employer is allowed to withhold: the full 0.18% family rate plus 40% of the 0.70% medical rate, which is 0.46% × $175,002 = $805.01, for $14,192.66. The question asks for 'mandatory employee state payroll taxes,' and Massachusetts lets the employer absorb some or all of the PFML share. That gives two readings: (1) count PFML at the maximum withholdable employee share, which is the engine's reading, or (2) leave it out because the employee is not required to pay it, which gives $13,387.65. The prompt does not settle this. Many models gave exactly $13,387.65 under reading (2). The other wrong answers come from separate mistakes: wrong PFML rates, cutting the FICA wage base by the employer-sponsored insurance (ESI) premiums, outdated wage bases, or submitting a number that doesn't match their own arithmetic. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)." +us,scenario_081,payroll_tax,39,prompt_ambiguity,state_local_rule,False,,"Federal FICA is simple here: 6.2% Social Security on all $175,002, since it is under the 2026 wage base, gives $10,850.12. Medicare at 1.45% gives $2,537.53, and no Additional Medicare Tax applies below $200,000. The reference then adds the Massachusetts PFML employee contribution of 0.46% ($805.01), which is the 0.18% family rate plus 40% of the 0.70% medical rate, on the engine's assumption that the employer withholds the maximum employee share. The prompt asks for 'mandatory employee state payroll taxes' but never says whether the employer passes the PFML share to the employee, so there are two readings: include the maximum $805.01 share ($14,192.66), or exclude it as employer-optional ($13,387.65). About a dozen models took the second reading. Most of the rest included PFML at a stale or wrong rate, or broke the FICA base or arithmetic. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)." us,scenario_081,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_081,state_income_tax_before_refundable_credits,44,reference_engine_defect,taxable_income_or_deductions,False,,"Massachusetts Part B income is wages of $175,002 plus $56 of bank interest, for $175,058. Three Part B items come off that: the $2,000 maximum deduction for Social Security/Medicare withholding, the renter deduction (50% of rent, capped at $4,000 after 2023) and the $4,400 single personal exemption. That leaves $164,658, and 5% of it is $8,232.90. Part A adds about $5.51 of tax on roughly $110 of dividends, for $8,238.41. Most models dropped the $2,000 payroll-tax deduction, used the old $3,000 rent cap or skipped the rent deduction, or took deductions Massachusetts doesn't allow (federal itemized deductions, employee business expenses, or subtracting employer insurance premiums from wages). A smaller group got Part B right but left out the roughly $8 of tax on the interest and dividends. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes. Reference is an engine defect; output excluded from scoring (M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21)." +us,scenario_081,state_income_tax_before_refundable_credits,45,reference_engine_defect,taxable_income_or_deductions,False,,"Massachusetts Part B income is wages of $175,002 plus $56 of interest, for $175,058. Two Part B deductions come off that: the $2,000 Social Security/Medicare deduction cap and the renter deduction, which is 50% of the $44,400 rent capped at $4,000. That leaves $169,058, and the $4,400 single exemption brings it to $164,658, which is $8,232.90 at 5%. Part A dividends of about $110 add $5.51 at 5%, and the short-term capital loss does not offset ordinary income or dividends. Most models got one or more of these steps wrong. Common errors were leaving out the $2,000 FICA deduction or the $4,000 rent deduction, using the old $3,000 rent cap, cutting wages by the $7,389 employer insurance premiums, deducting federal itemized items such as mortgage interest, or netting the $1,080 capital loss against income. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes. Reference is an engine defect; output excluded from scoring (M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21)." us,scenario_081,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_chip_eligible,3,llm_error,age_disability,False,,"The decisive rule in the engine trace is CHIP's age criterion: child1 is age 1 and falls outside the applicable CHIP age range. The alternative Medicaid pathway also returns ineligible, so neither route produces CHIP eligibility." us,scenario_082,child1_early_head_start_eligible,6,llm_error,categorical_eligibility,False,,"Early Head Start eligibility is a conjunction: the child must be under 3 (or the household must contain a pregnant person) AND the household must pass the means test — income at or below the federal poverty guideline, with limited over-income slots up to 130% FPL, or a categorical pathway such as TANF/SSI receipt, foster-care placement, or homelessness. Every wrong model treated the birth-to-3 age band as sufficient and either skipped or explicitly discarded the income screen. This household's countable income of ~$90,132 is more than four times the two-person poverty guideline (~$21k, ~$28k at 130%), no listed fact establishes TANF, SSI, foster care, or homelessness, and there is no pregnancy, so both the categorical and income conditions evaluate False and eligibility is No." us,scenario_082,child1_medicaid_eligible,6,llm_error,health_coverage,False,,"The trap is that New York's generous children's coverage above 154% FPL runs through Child Health Plus — the state's separate CHIP program — not through Medicaid: NY Medicaid covers a child age 1–5 only up to 154% FPL, and the 400–405% FPL figures the models cited are the CHIP ceiling. The second half of the trap is income: Medicaid MAGI here is about $117.8k (543% FPL for a two-person household) because the $33,000 long-term capital loss is limited to a $3,000 deduction against ordinary income and because $20,580 of qualified and non-qualified dividends, taxable interest, short-term gain, and the state tax refund all count, while the $21,208 ESI premium is not subtracted from MAGI. Every model either imported a CHIP-scale threshold into Medicaid, understated MAGI to roughly $90k–$99k, or asserted child eligibility with no income test at all. At 543% FPL the child clears no Medicaid category (medicaid_category = NONE), so is_medicaid_eligible is False." us,scenario_082,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_wic_eligible,2,llm_error,thresholds_rates,False,,"WIC requires both categorical eligibility and financial eligibility; being age 1 satisfies only the categorical test. The household's approximately $90,132 annual income exceeds the 2026 WIC limit of 185% of the federal poverty guideline for a two-person household, so the child is not eligible." -us,scenario_082,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The 2026 computation follows post-OBBBA law. AGI is $117,661: the net capital loss is capped at $3,000, the SALT refund is included, and the $8 traditional IRA deduction is zero because the head is an active 401(k) participant above the phase-out range. Taxable income takes off the $24,150 HoH standard deduction plus the new $700 charitable deduction for non-itemizers. The ordinary portion is taxed at the 2026 HoH brackets (10% to $17,700, 12% to $67,450, then 22%), and the $13,424 of qualified dividends at 15%. Then the $600 CDCC and the $2,200 CTC are subtracted. The closest misses all dropped the $700 non-itemizer charitable deduction (a $154 overstatement). Others used the $2,000 CTC, pre-OBBBA brackets or standard deduction, TCJA-sunset rules with personal exemptions, itemized deductions that counted employer-sponsored insurance premiums as medical expenses, or netted the full $30,711 capital loss." +us,scenario_082,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The 2026 computation follows the OBBBA version of the law. The TCJA structure is permanent, so there are no personal exemptions, the head-of-household standard deduction is $24,150, the child tax credit is $2,200, and the 10%/12% brackets top out at $17,700/$67,450. OBBBA also adds a deduction for non-itemizers' charitable gifts, worth the full $700 here. The other rules are that the net capital loss is capped at $3,000; the $8 traditional IRA deduction is fully phased out for a 401(k) participant at this AGI; and the employer-sponsored insurance premiums are neither subtracted from wages nor itemized as medical costs. That gives $92,811.12 of taxable income, $12,379.77 of regular tax, and $9,579.77 after the $600 CDCC and $2,200 CTC. The largest near-miss group ($9,734) omitted only the $700 non-itemizer charitable deduction ($154 at 22%). Other models used TCJA-sunset rules (personal exemptions, a $1,000 CTC), stale brackets or deduction amounts, the full capital loss, or itemized deductions inflated by the insurance premiums." us,scenario_082,federal_refundable_credits,8,llm_error,credit_phaseout,False,,"The separating step is that the Additional Child Tax Credit is a residual, not a per-child entitlement: the 2026 CTC of $2,200 is first applied nonrefundably against tax liability, and only the unused remainder — capped at $1,700 and further limited to 15% of earned income over $2,500 — becomes refundable. This head-of-household filer has roughly $117,650 of AGI (wages net of the $181 traditional 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capital-loss limitation on a $30,711 net loss and the $8 IRA deduction), which produces federal tax before credits many times larger than $2,200, so the entire credit is consumed nonrefundably and nothing spills into the refundable bucket. EITC is zero on income grounds and independently on the investment-income limit, and no American Opportunity Credit, Recovery Rebate Credit, or refundable payroll tax credit applies. Every wrong model quoted a statutory refundable ceiling as the answer instead of computing the leftover after the nonrefundable CTC." us,scenario_082,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_082,local_income_tax,6,llm_error,state_local_rule,False,,"The household facts give only state = NY and never say the household lives in New York City. The prompt says to treat any unlisted status input as false, so NYC residence is false. In New York, only NYC residence triggers the separate local income tax output, so local_income_tax is $0. Every wrong model assumed NYC residence and ran NYC resident brackets on NY taxable income, which turned a state-only fact pattern into an NYC tax bill." -us,scenario_082,payroll_tax,43,prompt_ambiguity,state_local_rule,False,,"The answers split on whether New York's employee Paid Family Leave and disability (DBL) contributions count. The reference adds both at their statutory maximums: PFL at 0.432% of $100,195, capped at the 2026 maximum of $411.91, and DBL at $0.60 a week, or $31.20 a year. That puts $443.11 on top of FICA of $7,664.92. New York lets employers collect less or none, so the prompt's phrase ""mandatory employee state payroll taxes"" supports two readings: (1) the statutory maximum employee PFL/DBL contributions count, or (2) they are not mandatory employee taxes, which leaves FICA only. Eighteen models took reading (2). The rest included New York items but used old PFL rates or caps (0.373% or 0.388%, maximums of $333–$370), skipped the $411.91 cap, left out PFL or DBL, or cut FICA wages by subtracting the ESI premiums or the 401(k) deferral. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)." +us,scenario_082,payroll_tax,44,prompt_ambiguity,state_local_rule,False,,"Federal FICA on the full $100,195 of gross wages is $7,664.92 (SS $6,212.09 + Medicare $1,452.83). Neither the traditional 401(k) deferral nor the ESI premium input reduces that base. The reference adds the maximum New York employee contributions: Paid Family Leave at 0.432% capped at $411.91, and DBL at $0.60/week, or $31.20. That brings the total to $8,108.03. The prompt says ""mandatory employee state payroll taxes,"" and New York lets the employer collect less of these contributions or none, so FICA-only answers of $7,664.92 follow a second reasonable reading. Other answers differed from the frozen reference by using outdated or uncapped PFL rates, including only PFL or DBL, or reducing the engine's FICA wage base by ESI premiums or 401(k) deferrals; omitting either optional New York contribution is not itself a model error. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)." us,scenario_082,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_082,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"Getting the reference takes four New York-specific steps. First, find NY AGI: subtract the $68 SALT refund from federal AGI. Federal AGI caps the capital loss at $3,000 and doesn't include the ESI premiums. Second, use head-of-household status, which gives an $11,200 standard deduction (more than the $9,328 of NY itemized deductions) plus a $1,000 dependent exemption, for $105,393.62 of taxable income. Third, apply the reduced 2026 HOH schedule (3.9%/4.4%/5.15%/5.4%) to get $5,442.63, then add $156.51 of supplemental recapture tax because NY AGI exceeds $107,650. Fourth, subtract no nonrefundable credits: the NY child-care credit and the Empire State Child Credit are refundable, and the household credit is phased out at this income. The models went wrong on filing status, the dependent exemption, the capital-loss cap, the ESI premiums, the pre-2026 rates, the omitted supplemental tax, or by treating the refundable NY child-care credit as nonrefundable." -us,scenario_082,state_refundable_credits,47,llm_error,state_local_rule,False,,"Two NY rules make up the $1,187.55. First, the enhanced Empire State Child Credit gives $1,000 for a child under 4. It is cut by $16.50 for each full $1,000 of federal AGI above the $75,000 head-of-household threshold: 42 full increments = $693, leaving $307. Second, the 2026 child care credit under Tax Law 606(c-2) is no longer a percentage of the federal CDCC. It applies an AGI-based percentage (about 29.352% at NY AGI of $117,594) directly to the $3,000 of qualifying expenses, giving $880.55 of refundable credit. Nearly every model used the old rule (20% of the $600 federal credit, or $120), said the care credit was nonrefundable, or used the pre-2025 ESCC rules: the $100/33% formula, under-4 children excluded, or an unphased $1,000." +us,scenario_082,state_income_tax_before_refundable_credits,48,llm_error,state_local_rule,False,,"The correct path files head of household. Federal AGI is $117,661, using wages net of the 401(k), the $3,000 capital-loss cap, no deductible IRA and no reduction for ESI premiums. New York subtracts the $67.50 state tax refund, takes the $11,200 HOH standard deduction because NY itemized deductions are only $9,328, and subtracts one $1,000 dependent exemption, for taxable income of $105,394. The 2026 reduced-rate HOH schedule (3.9/4.4/5.15/5.4%) gives $5,442.63. Because NY AGI exceeds $107,650, a $156.51 supplemental recapture tax applies, and no nonrefundable credit reduces the result, since the NY child and dependent care credit and the Empire State Child Credit are refundable. Most models went wrong on one of four points: they left out the recapture, used pre-2026 rates, used the wrong filing status or deduction (single $8,000, inflated itemized deductions, ESI premiums subtracted, uncapped capital loss), or subtracted refundable credits." +us,scenario_082,state_refundable_credits,48,llm_error,state_local_rule,False,,"To get the reference, a model needed two current New York rules. The first is the enhanced Empire State Child Credit: $1,000 for a child under 4, reduced by $16.50 per $1,000 of federal AGI above the $75,000 head-of-household threshold (42 × $16.50 = $693), which leaves $307. The second is the new child and dependent care credit under Tax Law 606(c-2), which applies from 2026: an applicable percentage of about 29.35% at NY AGI of roughly $117.6k is applied directly to the $3,000 of qualifying expenses, giving $880.55. Almost every model either used the old ESCC formula ($100 minimum or 33% of the federal CTC), skipped the phase-out, or zeroed out the ESCC. Almost all also used the pre-2026 care credit (20% of the $600 federal CDCC = $120) or treated it as nonrefundable." us,scenario_083,federal_income_tax_before_refundable_credits,1,llm_error,thresholds_rates,False,,"The 2026 single-filer standard deduction exceeds the household’s $11,010 AGI, leaving zero taxable income. With no other federal tax components or nonrefundable credits affecting the result, federal income tax before refundable credits is $0." us,scenario_083,federal_refundable_credits,9,llm_error,categorical_eligibility,False,,"A childless filer must satisfy the EITC age requirement, and the 20-year-old head does not. With no qualifying children and no applicable refundable education, recovery rebate, or payroll tax credit, every component of federal refundable credits is zero." us,scenario_083,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Texas did not adopt ACA Medicaid expansion, so income below 138% of FPL does not create Medicaid eligibility for a nondependent adult. The 20-year-old head has no qualifying Texas category—such as pregnancy, parent/caretaker status, disability, or SSI eligibility—so `medicaid_category` is NONE regardless of the 0.69 FPL MAGI level." @@ -662,7 +662,7 @@ us,scenario_083,payroll_tax,6,llm_error,other,False,,"The correct answer is the us,scenario_083,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Ordinary rental income is generally excluded from net earnings from self-employment and therefore does not enter the self-employment tax base. With no reported self-employment earnings, the $400 filing threshold is not met and self-employment tax is zero." us,scenario_083,snap,18,llm_error,asset_resource,False,,"This one-person Texas household passes every SNAP income test: gross income is $917.50 a month, well under 130% and 165% of FPL, and net income is about $591, under 100%. What makes it ineligible is its resources. Texas's broad-based categorical eligibility keeps a $5,000 asset limit, and the federal standard for a non-elderly household is about $3,000. The head's $6,620 bank balance exceeds both, so SNAP is $0. Every model ran the income-based benefit formula, getting about $110-$245 a month. None applied the resource test, and several said outright that Texas BBCE waives the asset test." us,scenario_084,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Disability, blindness, SSDI receipt, and qualifying immigration status do not independently establish Medicaid eligibility; the person must satisfy a specific North Carolina coverage pathway and its financial rules. The head qualifies through no Medicaid category, and Medicaid MAGI is 1.56 times FPL, above the expansion threshold." -us,scenario_084,head_medicare_eligible,34,prompt_ambiguity,age_disability,False,,"The head is 62, so they don't meet Medicare's age-65 test. Under 65, PolicyEngine grants Medicare eligibility only after at least 24 months of Social Security disability receipt, or for ESRD. The household facts never give how many months the head has received SSDI. The prompt says unlisted numeric inputs are 0, so the months of SSDI receipt are 0 and the 24-month waiting period is not met. Every model treated being disabled plus current SSDI income as Medicare eligibility, or assumed the 24 months were already served. That assumption broke the prompt's explicit rule against inferring unlisted facts. Even counting all 12 months of SSDI in the tax year would still fall short of 24. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_084,head_medicare_eligible,35,prompt_ambiguity,age_disability,False,,"PolicyEngine's is_medicare_eligible is true only if the person is 65 or older, has ESRD, or has months_receiving_social_security_disability of at least the 24-month waiting period. The Head is 62. No SSDI duration is listed, so under the prompt's rule that unlisted numeric inputs are 0, the SSDI month count is 0. Being disabled and receiving SSDI in 2026 therefore does not meet the disability pathway, and the Head is not eligible. Every model treated disability status plus SSDI receipt as enough, or assumed the 24-month waiting period was already met, which the prompt never states. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_084,snap,2,llm_error,thresholds_rates,False,,"The single-person household has $33,233 of gross annual income ($2,769/month), which exceeds North Carolina's broad-based categorical eligibility gross cap of 200% FPL (about $2,680/month for one person in 2026), and after the ~$204 standard deduction and the elderly/disabled excess-medical deduction its net income of roughly $1,738/month still exceeds the 100%-FPL net income limit of about $1,340/month; the $56,500 bank balance also exceeds the elderly/disabled resource limit of roughly $4,650. Ineligibility ends the calculation at $0. Both models stopped at the observation that the gross income test is waived when an elderly or disabled member is present and never tested net income against the 100%-FPL ceiling, then manufactured a benefit — one by overriding its own $0 formula result, the other by applying the one-to-two-person minimum allotment, which is a floor for households already found eligible, not a substitute for eligibility." us,scenario_085,federal_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"The correct path starts with taxable Social Security of about $14,051, which is $4,500 plus 85% of provisional income above $34,000. That gives AGI of about $45,574. Under OBBBA, the TCJA structure is permanent, so 2026 has no personal exemption and a $16,100 single standard deduction plus $2,050 for being 65 or older, for $18,150. The $6,000 OBBBA senior deduction also applies because MAGI is under $75,000, bringing total deductions to $24,150. Taxable income is $21,423.74, taxed at 10% up to $12,400 and 12% above that, for $2,322.85, and there are no nonrefundable credits. Most models either left out the $6,000 senior deduction, assumed TCJA had expired (personal exemption and a 15% bracket), miscalculated taxable Social Security, or submitted a number that didn't match their own correct work." us,scenario_085,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -670,15 +670,15 @@ us,scenario_085,local_income_tax,1,parse_contract_failure,missing_output,False,, us,scenario_085,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered employment earnings, not Social Security survivor benefits, retirement distributions, pension income, or taxable interest. Because the household has no listed wages or other payroll-taxable earnings, the employee-side payroll-tax base is zero and payroll tax is $0." us,scenario_085,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_085,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_085,state_income_tax_before_refundable_credits,34,llm_error,state_local_rule,False,,"This 67-year-old's Social Security and retirement distributions are exempt from PA tax, so PA taxable income is just the $1,675 of interest. At 3.07% that is $51.42 of tax. Pennsylvania's Special Tax Forgiveness (Schedule SP) then removes it entirely. Eligibility income for forgiveness leaves out Social Security and commonly recognized retirement benefits, so it is only $1,675, well under the $6,500 single-filer limit for 100% forgiveness. Most models stopped at $51.42, either never applying forgiveness or rejecting it because they counted the $27,426 of Social Security and $29,848 of retirement income as eligibility income. A few taxed the retirement distributions outright." +us,scenario_085,state_income_tax_before_refundable_credits,35,llm_error,state_local_rule,False,,"Pennsylvania does not tax Social Security or retirement distributions (pension, IRA, 401(k)) received after retirement age, so this 67-year-old's only PA-taxable income is $1,675 of interest, which gives a gross tax of $51.42. The Tax Forgiveness credit (Schedule SP) is nonrefundable, and its eligibility income also leaves out Social Security and those retirement distributions. That puts eligibility income near $1,675, well under the $6,500 single-filer limit for 100% forgiveness, so the liability goes to $0. Most models stopped at $51.42 and skipped forgiveness, often because they counted Social Security and pensions toward the forgiveness limit. A smaller group taxed the retirement distributions, which PA exempts." us,scenario_086,child1_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires Medicaid ineligibility and satisfaction of CHIP’s independent age and income criteria. Child 1 is age 12, so age is not the failing criterion; the household income exceeds Georgia’s applicable CHIP income threshold for 2026." us,scenario_086,child1_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The models conflated Georgia’s separate PeachCare/CHIP pathway with Medicaid or assumed that being a child creates Medicaid eligibility at higher income levels. At 3.44 times FPL, child1 exceeds Georgia’s child Medicaid thresholds and qualifies for no Medicaid category, so employer-sponsored coverage and non-MAGI expense deductions do not create eligibility." -us,scenario_086,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,"PolicyEngine allows a $337.50 above-the-line educator expense deduction for 2026. IRC §62(a)(2)(D) and §62(d)(3) index the $250 base and round it down to a multiple of $50, which gives a 2026 limit of $300. The engine appears to uprate the cap without that round-down, which overstates the deduction by $37.50 and understates tax by $4.50. The statutory value would be $3,260.05.","The $12,824 of Social Security dependent benefits is listed on Head, so it counts as Head's income. Provisional income is about $68,075, well above the $34,000 HoH adjusted base, so 85% ($10,900.40) is taxable and AGI is $72,562.90. In 2026 the OBBBA-extended TCJA rules apply: a $24,150 HoH standard deduction, a 10% bracket up to $17,700 and a 12% bracket above it, no personal exemptions, and a $2,200 CTC used in full against $5,455.55 of tax. The household facts do not support a car-loan interest deduction. Most wrong answers left out the taxable Social Security, used TCJA-sunset rules (exemptions, 15% bracket, $1,000 CTC), used an older standard deduction or bracket threshold, or deducted the $275 auto-loan interest. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)." +us,scenario_086,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,"PolicyEngine deducts the full $337.50 educator expense above the line instead of capping it at the IRC §62(a)(2)(D)/(d)(3) limit of $300 per educator. That limit is $250 indexed and rounded down to a $50 multiple, so it is still $300 for 2026. The excess deduction understates taxable income by $37.50 and tax by $4.50; the capped result is about $3,260.05.","The reference files head of household and attributes the $12,824 of Social Security dependent benefits to the head. Provisional income is about $68,075, far above the $34,000 upper base, so 85% ($10,900.40) is taxable. It then subtracts $337.50 of educator expenses and OBBBA's permanent 2026 HoH standard deduction of $24,150, taxes $48,412.90 at 10% up to $17,700 and 12% above that ($5,455.55), and subtracts the full $2,200 CTC. The models went wrong in five ways: treating the benefits as the child's non-taxable income, assuming TCJA expired (personal exemptions, 15% bracket, $1,000 CTC), using stale standard deduction and CTC amounts ($22,500 to $23,350 and $2,000), using single or MFJ status, or deducting the $275 of auto loan interest, which the engine does not allow here. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)." us,scenario_086,federal_refundable_credits,11,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit is a residual, not an entitlement: IRC 24(d)(1) makes it the lesser of the per-child refundable cap ($1,700 for 2026) and the amount of CTC that the §26(a) tax-liability limitation actually blocked, which Schedule 8812 computes as line 12 minus line 14 (""If zero, stop here; you cannot take the additional child tax credit""). This head-of-household filer has AGI of $72,562.90 ($62,000 wages plus $10,900.40 of the $12,824 Social Security dependent benefit that is taxable once combined income exceeds the $34,000 tier-2 threshold, less the $337.50 educator deduction), taxable income of $48,412.90 after the 2026 HoH standard deduction of $24,150, and pre-credit tax of $5,455.55 — so the full $2,200 CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is $0 because earned income of $62,000 is far past the 2026 one-child HoH phase-out endpoint of roughly $51,600, computed at the statutory 15.98% phase-out rate, and no education expenses, rebate, or refundable payroll credit exist here. Ten of eleven models treated the $1,700 refundable cap (or a stale $1,000/$1,500 figure) as a floor paid out per qualifying child and never performed the unused-CTC subtraction against liability." us,scenario_086,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_086,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_086,payroll_tax,8,llm_error,payroll_tax_base,False,,"Employee payroll tax is $3,844 of Social Security tax plus $899 of Medicare tax on the head’s full $62,000 wage amount, totaling $4,743. The listed insurance premiums do not reduce the FICA wage base in this computation, Georgia unemployment insurance is employer-paid rather than a mandatory employee payroll tax, and the dependent child has no wages." -us,scenario_086,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Georgia AGI is federal AGI ($72,562.90, which includes $10,900.40 of taxable Social Security) minus the full taxable Social Security subtraction, giving $61,662.50. For 2026, PolicyEngine then applies Georgia's $15,000 head-of-household standard deduction and a $5,000 dependent exemption, with no separate personal exemption. That leaves taxable income of $41,662.50, taxed at the flat 4.99% rate, for $2,078.96. Nearly every model kept older Georgia parameters: a $12,000/$18,500/$24,000 standard deduction, $3,000/$4,000 dependent exemptions, pre-2024 $2,700 personal exemptions, or a 5.09–5.49% rate. Several also mishandled the Social Security subtraction or employer health premiums." +us,scenario_086,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"Georgia AGI starts from federal AGI ($72,562.90, which includes $10,900.40 of taxable Social Security). Subtracting that taxable Social Security gives $61,662.50. PolicyEngine then takes the $15,000 head-of-household standard deduction and a $5,000 dependent exemption, leaving $41,662.50 of taxable income, and applies the 2026 flat rate of 4.99% to get $2,078.96. Almost every model used an outdated rate (5.09%, 5.19%, 5.29%, 5.39% or 5.49%), the $12,000 single deduction with a $3,000 or $4,000 dependent exemption, or the personal exemptions that existed before HB 1437. Several also mishandled the Social Security subtraction or the federal AGI starting point." us,scenario_086,state_refundable_credits,1,llm_error,categorical_eligibility,False,,"Georgia does not provide the claimed $260 refundable credit for each dependent child under age 17. With no applicable Georgia refundable credit, the state-level refundable-credit aggregate is $0." us,scenario_088,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The decisive computation step is the 2026 deduction for qualified tip income: the $9,600 of tips included in wages reduces taxable income in addition to the applicable standard deduction. The state income-tax refund is also excluded because no prior-year itemized-deduction tax benefit is established. These deductions reduce taxable income to zero, so federal income tax before refundable credits is $0 and cannot be negative." us,scenario_088,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The only refundable credit in play is the childless EITC, and IRC §32(a)(2)(B) phases it out against the GREATER of earned income or AGI. Earned income is $18,577, but AGI is $24,035 once the $2,733 unemployment compensation, $1,862 taxable pension, $675 state tax refund, $140 interest and $48 dividends are added; the tip deduction is below-the-line and does not reduce AGI. The 2026 childless EITC completes its phase-out below $20,000, so an AGI of $24,035 zeroes the credit, and with no qualifying children, no education expenses and no rebate, every other refundable component is $0. Every wrong model ran the phase-out on earned income alone, or on a max-credit shortcut, and never applied the AGI leg of the test." @@ -697,7 +697,7 @@ us,scenario_090,head_medicaid_eligible,6,llm_error,categorical_eligibility,False us,scenario_090,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires categorical eligibility in addition to income eligibility and nutritional risk. A 22-year-old adult with no pregnancy, postpartum, or breastfeeding status is outside every eligible WIC category, so passing the other two tests does not confer eligibility." us,scenario_090,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security tax plus 1.45% Medicare tax on the full $7,000 of wages, with no Additional Medicare Tax or mandatory Kansas employee payroll tax. The components are $434.00 and $101.50, which sum to $535.50." us,scenario_090,snap,8,llm_error,asset_resource,False,,"Kansas does not use broad-based categorical eligibility (BBCE) for SNAP, so the federal resource test applies. For a household with no elderly or disabled member, the limit is $3,000. The head's $20,300 bank balance exceeds that limit several times over, so SNAP is $0 however the income side works out. Every model produced a positive benefit, either by ignoring the $20,300 in assets or by wrongly claiming that Kansas BBCE waives the asset test." -us,scenario_090,state_income_tax_before_refundable_credits,20,llm_error,taxable_income_or_deductions,False,,"Kansas's 2024 tax law (SB 1) replaced the old $2,250 per-person exemption with a $9,160 personal exemption for a single filer, on top of the $3,605 single standard deduction. Together they shelter $12,765, which is more than this filer's $7,020 Kansas AGI, so Kansas taxable income is $0 and so is the tax. Every model used the pre-2024 exemption ($2,250), the dependent-sized $2,320 figure, a $110 credit, or no exemption at all, which left $900 to $3,520 of phantom taxable income. Several models then made it worse with retired rates (3.1%, 5.15%, 5.5%, 5.7%), but the exemption error alone produces the nonzero answer." +us,scenario_090,state_income_tax_before_refundable_credits,21,llm_error,taxable_income_or_deductions,False,,"Kansas SB 1 (2024) raised the personal exemption for single filers from $2,250 to $9,160, effective tax year 2024. It also replaced the three brackets with two rates of 5.2% and 5.58%. For a single filer, the inflation-indexed standard deduction (about $3,605 or more) plus the $9,160 exemption totals well over $12,700. That far exceeds this filer's $7,020 Kansas AGI, so Kansas taxable income is $0 and tax before refundable credits is $0. Every model used the old $2,250 or $2,320 exemption, or left the exemption out entirely, which left $900–$3,520 of taxable income." us,scenario_090,state_refundable_credits,5,llm_error,categorical_eligibility,False,,"Kansas's refundable individual credit is its EITC, set at 17% of the federal EITC, and the head is 22 — below the age-25 floor for the childless federal EITC — so the federal credit is $0 and the state match on it is $0. The other candidate, the Kansas food sales tax credit, is nonrefundable for tax years after 2012 and in any case requires the filer to be 55 or older, blind or disabled, or to have a dependent child under 18 in the home, none of which describes a 22-year-old single filer with no dependents. Every wrong answer cleared a household income test ($7,020 is well under both credits' ceilings) and then stopped, never applying the categorical age/dependent gate that actually decides both credits. The correct derivation is 17% × $0 federal EITC plus $0 qualifying food sales tax credit = $0." us,scenario_090,tanf,1,llm_error,categorical_eligibility,False,,"TANF is a benefit for families with dependent children; a single 22-year-old with no children fails Kansas's categorical requirement outright, and the $20,300 in bank assets independently exceeds the state's TANF resource limit, so the correct answer is $0. The trap is the ""financial assistance"" line item in the household facts: it is an unearned-income input, not a TANF payment, and treating it as the answer skips the eligibility determination entirely. Answering this variable requires running the categorical test — dependent child present, resources under the limit — rather than pattern-matching a dollar figure labeled ""assistance"" in the inputs." us,scenario_091,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"The reference uses post-OBBBA 2026 law. AGI is $35,012.56 (wages net of the $926 traditional 401(k), plus interest and dividends, minus the $43 IRA deduction). Two deductions come off that: the $16,100 standard deduction and the $628 qualified-overtime deduction, which gives taxable income of $18,284.66. After the $3,968 of qualified dividends is pulled out at 0%, ordinary income of $14,316.66 is taxed at 10% up to $12,400 and 12% above it, for $1,470. A $120 nonrefundable Saver's Credit (10% of $1,199 in traditional plus Roth contributions) brings it to $1,350. The AOTC is $0 because no qualified tuition is listed, and ESI premiums are not excluded from wages. Most wrong answers made one of these mistakes: dropped the overtime deduction, dropped the Saver's Credit or cut its base, claimed an AOTC with no expenses, assumed the TCJA sunset, excluded the $6,589 of ESI premiums from wages, or used outdated standard-deduction and bracket amounts." @@ -705,7 +705,7 @@ us,scenario_091,federal_refundable_credits,10,llm_error,other,False,,"The decisi us,scenario_091,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Wisconsin Medicaid eligibility requires qualification through an applicable categorical pathway and its income rules. This 22-year-old nondependent falls into no Medicaid category, and MAGI of 2.19 times FPL does not satisfy an adult eligibility pathway, so immigration eligibility alone does not confer coverage." us,scenario_091,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 6.2% Social Security plus 1.45% Medicare on the full $27,000 of gross wages, which comes to $1,674.00 + $391.50 = $2,065.50. The listed $6,589 of employer-sponsored insurance premiums and the $1,800 of other health premiums do not lower the FICA wage base. The $628 FLSA overtime premium is already inside the $27,000 gross, and Wisconsin has no mandatory employee state payroll tax. Most of the wrong models treated the ESI premiums as a pre-tax Section 125 deduction and taxed only $20,411. The rest added the overtime twice, invented a state payroll tax, used the wrong combined rate, or made an arithmetic or reporting slip." us,scenario_091,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_091,state_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,,"The engine starts from Wisconsin AGI of $35,012.56. That is wages net of the $926.16 pre-tax 401(k) deferral, plus $4,806 of interest and $4,176 of dividends, minus the $43.28 IRA deduction. The $1,170 of non-Schedule-D capital gain distributions is left out entirely. Employer insurance premiums, other health premiums and overtime are not subtracted. It then subtracts the sliding-scale standard deduction of $12,067.66 and the $700 personal exemption, which gives $22,244.90 of taxable income, taxed at 3.5%/4.4% to get $843.66 with no nonrefundable credits. The models went wrong in four ways: (1) they included the $1,170 of gains, in full or at 70%; (2) they phased the standard deduction down too far, left it unphased, or dropped the $700 exemption; (3) they subtracted the $6,589 of employer insurance premiums or the $1,800 of health premiums from income; or (4) they invented nonrefundable credits (AOTC, school property tax credit) that cut the tax to $0. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)." +us,scenario_091,state_income_tax_before_refundable_credits,48,llm_error,state_local_rule,False,,"The Wisconsin calculation has three parts. Start from federal AGI of $36,182.56. That figure includes all $10,152 of interest, dividends and capital gain distributions, and it keeps the employer-sponsored insurance premiums, the other health premiums and the overtime premium in income. Subtract 30% of the $1,170 long-term capital gain distributions ($351) to get WI AGI of $35,831.56. Then deduct the indexed 2026 sliding-scale standard deduction, which is only partly phased out at $12,074.61, plus the $700 exemption, which is a deduction and not a credit. Tax the resulting $23,056.95 at 3.5% up to roughly $15,110 and 4.4% above that, with no nonrefundable credit. Most models went wrong in one of these ways: they skipped the capital gain exclusion, phased the standard deduction down too steeply or not at all, dropped the exemption, or subtracted items such as ESI premiums or overtime that never reduce WI income. A few zeroed out the tax with credits that do not apply." us,scenario_091,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_092,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Only the $14,980 taxable pension enters adjusted gross income; the Social Security benefit contributes no taxable amount under the federal provisional-income calculation. The age-77 single filer’s applicable standard deduction exceeds that AGI, leaving zero taxable income and therefore zero federal income tax before refundable credits." us,scenario_092,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and disability do not independently establish Alabama Medicaid eligibility; the person must qualify through a specific Medicaid category. This person receives no SSI, has MAGI income of 2.03 times FPL, and qualifies through none of the available pathways, so the Medicaid category is NONE and eligibility is No." @@ -715,10 +715,10 @@ us,scenario_092,state_income_tax_before_refundable_credits,23,llm_error,taxable_ us,scenario_092,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_093,dependent1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not confer Missouri Medicaid eligibility. The person must satisfy a specific non-MAGI pathway's categorical and financial requirements; this 27-year-old dependent receives no SSI, qualifies for no non-MAGI category, and at 2.82 times FPL exceeds the applicable MAGI limits." us,scenario_093,dependent1_medicare_eligible,5,llm_error,age_disability,False,,"PolicyEngine's is_medicare_eligible turns on a single age test — age 65 or older — with the narrow ESRD/ALS entitlements not modeled and no disability branch at all; Dependent 1 is 27, so the variable is False. Every wrong model substituted the household's bare ""is disabled"" flag for that age test, asserting a disability-based Medicare pathway either as a PolicyEngine rule or as a real-world shortcut. The statutory disability pathway requires 24 months of SSDI entitlement, and Dependent 1 has $45,000 in annual wages (~$3,750/month), far above substantial gainful activity, plus no listed SSDI receipt — the prompt directs unlisted benefit receipt to be treated as zero, so no disability pathway is reachable on these facts." -us,scenario_093,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,household_unit_or_filing_status,False,,"The joint return counts only the head's and spouse's income. Wages of $118,000, minus $4,058 of traditional 401(k) deferrals, plus $1,156 of interest, minus a $190 traditional IRA deduction and $2,500 of capped student loan interest, give AGI of $112,408. The 2026 standard deduction of $32,200 leaves $80,208 of taxable income, taxed at 10%/12% for $9,129, and the $500 credit for other dependents for the disabled 27-year-old brings it to $8,629. The disabled adult is a qualifying child of any age, so no gross-income or support test removes the credit. Most models went wrong in one of these ways: they put the dependent's $45,000 wages on the joint return, added a separate tax on the dependent, denied the $500 credit, used TCJA-sunset or 2025 parameters, subtracted ESI premiums from wages, or took an auto-loan interest deduction that doesn't apply. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." +us,scenario_093,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,household_unit_or_filing_status,False,,"The correct figure comes from one married-filing-jointly return that taxes only the head's and spouse's income. Wages of $118,000 minus $4,058 in traditional 401(k) contributions, plus $1,156 of interest, minus $190 in IRA contributions and the $2,500 student-loan-interest cap gives AGI of $112,408. Subtracting the 2026 OBBBA standard deduction of $32,200 leaves taxable income of $80,208, which produces $9,129 of tax at 10% up to $24,800 and 12% above that. The disabled 27-year-old is claimed as a qualifying child, because permanent disability removes the age limit, so the couple gets the $500 credit for other dependents, for a final $8,629. That dependent's $45,000 of wages is neither added to the joint AGI nor taxed on a separate return in this output. Most wrong answers either added the dependent's wages or a separate return, denied the $500 credit, subtracted employer health insurance premiums from wages, took an unsupported auto-loan-interest deduction, or used pre-OBBBA or TCJA-sunset parameters. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." us,scenario_093,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_093,payroll_tax,19,llm_error,payroll_tax_base,False,,"Employee FICA here is 7.65% of each worker's full listed gross wages: 6.2% Social Security plus 1.45% Medicare on $53,000, $65,000 and $45,000, all below the wage base. That gives $3,286 + $768.50, $4,030 + $942.50 and $2,790 + $652.50, for a total of $12,469.50. No Additional Medicare Tax applies, and Missouri has no employee state payroll tax. The facts never say the $8,389 ESI premiums were paid through a pre-tax cafeteria plan. The largest group of models (8) assumed a Section 125 salary reduction anyway, cutting FICA wages to $137,833 and the tax to $10,544.22. Most other models wrote out the correct $12,469.50 components or formula but then added wrong, multiplied wrong, or submitted a different number from the one they computed." -us,scenario_093,state_income_tax_before_refundable_credits,47,prompt_ambiguity,household_unit_or_filing_status,False,,"The engine puts the 27-year-old disabled dependent in the couple's tax unit with $0 Missouri AGI. Only the spouses' $112,408.25 is taxed. That figure reflects excluding 401(k) deferrals, the $190 IRA deduction and the $2,500 student-loan-interest cap. Missouri taxable income is $79,776.80 after the $32,200 2026 MFJ standard deduction and a federal-tax deduction of 5% of federal tax ($431.45), which applies at MO AGI of $100,001–$125,000. That income is split between the spouses by income share, and the 4.7%-top-rate schedule is applied to each share separately, so each spouse gets the lower-bracket savings. Most wrong answers taxed the dependent's $45,000 of wages, either in joint AGI or on a separate single return, which pushed AGI past $125,000 and eliminated the federal-tax deduction. Other wrong answers used made-up personal exemptions, outdated rates or deductions, subtracted employer insurance premiums from wages, or applied the schedule only once to the joint total. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." +us,scenario_093,state_income_tax_before_refundable_credits,48,prompt_ambiguity,household_unit_or_filing_status,False,,"Missouri tax here comes only from the spouses' income. Their MO AGI is $112,408.25, and the dependent's $45,000 of wages adds $0. Subtracting the $32,200 2026 standard deduction and a $431.45 federal-income-tax deduction leaves $79,776.80. The federal-tax deduction is 5% of the couple's $8,629 federal tax because MO AGI falls in the $100,001–$125,000 tier. Missouri's combined return then splits that taxable income between the spouses by income share ($34,743.94 and $45,032.86) and runs each share through the graduated 2026 schedule (4.7% top rate) separately, so the lower brackets apply twice. Most models either added the dependent's wages to the base or taxed a separate dependent return, which also pushed AGI past $125k and wiped out the federal-tax deduction. Nearly all of them ran the schedule once on the joint total instead of separately for each spouse. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." us,scenario_095,federal_income_tax_before_refundable_credits,3,llm_error,household_unit_or_filing_status,False,,"The head's surviving-spouse status determines the federal filing-status pathway and supplies a standard deduction large enough to eliminate the $15,286.89 of AGI. Social Security is not taxable at this provisional-income level, so it is excluded from AGI; after the half-self-employment-tax deduction, taxable income and income tax before credits are both zero." us,scenario_095,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"Federal refundable credits include only the refundable credit components applicable under 2026 law. Age 73 does not independently create a $1,200 refundable senior credit, and all five applicable components—EITC, refundable American Opportunity Credit, refundable CTC, Recovery Rebate Credit, and refundable payroll tax credit—equal zero." us,scenario_095,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -730,7 +730,7 @@ us,scenario_095,payroll_tax,2,llm_error,payroll_tax_base,False,,"The payroll_tax us,scenario_095,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_095,self_employment_tax,7,llm_error,payroll_tax_base,False,,"The correct method has two steps. First, reduce self-employment income to net earnings from self-employment by multiplying by 0.9235, which gives about $3,877. That amount is above the $400 filing floor and well below the Social Security wage base. Second, apply the full 15.3% rate, which gives about $593. Some models left out the 0.9235 adjustment. Others compared the wrong figure to the $400 floor or treated the income-tax deduction for half of SE tax as a reduction of SE tax itself. Others did the arithmetic and then submitted a different number. The head's age of 73 and receipt of Social Security do not exempt them from SE tax." us,scenario_095,snap,5,llm_error,thresholds_rates,False,,"The household’s $29,605 annual income exceeds the applicable 2026 New Jersey SNAP gross-income eligibility limit for one person, so the SNAP calculation stops with no eligibility and a $0 allotment. Broad-based categorical eligibility, elderly-household deductions, and the minimum-allotment rule do not produce a benefit after this income screen fails." -us,scenario_095,state_income_tax_before_refundable_credits,22,llm_error,thresholds_rates,False,,"New Jersey imposes no income tax when NJ gross income, which excludes Social Security and is measured after the age-62+ pension exclusion, is $10,000 or less for single filers, or $20,000 or less for qualifying widow(er)s and the other joint-type statuses. For this 73-year-old, NJ gross income is $15,583 minus the $11,385 401(k) exclusion, or $4,198. That is below either threshold, so the tax is $0. Almost every model correctly excluded Social Security and the 401(k) distribution. They then treated the leftover $4,198 of self-employment income as taxable, took off exemptions and sometimes medical expenses, and applied the 1.4% bracket. None applied the gross-income threshold that zeroes the liability." +us,scenario_095,state_income_tax_before_refundable_credits,23,llm_error,state_local_rule,False,,"New Jersey charges no income tax when NJ gross income is at or below the filing threshold: $10,000 for single filers and $20,000 for a qualifying widow(er). Social Security is exempt, and the $11,385 401(k) distribution is removed by the age-62+ pension exclusion. That leaves $4,198 of NJ income, far below either threshold, so the tax is $0 before any rate schedule applies. Most models got the exclusions right. They then ran the leftover self-employment income through exemptions, the medical deduction and the 1.4% bracket, and never checked the no-tax threshold." us,scenario_095,state_refundable_credits,4,llm_error,credit_phaseout,False,,"This household has no children, so the NJ EITC is its only possible refundable credit. For a childless filer aged 65 or older, New Jersey computes the credit as 40% of a hypothetical federal childless EITC. The phase-in gives 7.65% × earned income of about $3,901 ($4,198 SE income minus $296.58, the deductible half of SE tax), or $298.45. The phase-out then takes 7.65% × (AGI − non-joint threshold). AGI is about $15,286 because it includes the $11,385 401(k) distribution (Social Security is untaxed because provisional income is below $25,000). Since the 401(k) distribution alone exceeds the roughly $11k threshold, the reduction is larger than the phase-in amount and the credit is $0. The household paid no property tax or rent, so the NJ property tax credit is also $0, and ANCHOR is not an income tax credit." us,scenario_095,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_098,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The $11,100 farm rental loss offsets the $15,540 taxable pension, leaving $4,440 of non-Social-Security income. Provisional income is then below the threshold for taxing Social Security benefits, so AGI remains $4,440; the applicable standard deduction eliminates taxable income and produces zero federal income tax before refundable credits." @@ -757,15 +757,15 @@ us,scenario_099,child2_head_start_eligible,2,llm_error,categorical_eligibility,F us,scenario_099,child2_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility requires both categorical eligibility and financial eligibility; being age four satisfies only the child-category requirement. The household's approximately $159,676 of annual income exceeds California's WIC limit of 185% of the federal poverty level for a four-person household, so child 2 is not eligible." -us,scenario_099,federal_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"For 2026, the reference uses permanent OBBBA law. Under it, the MFJ standard deduction is $32,200 and non-itemizers also get a $2,000 charitable deduction. The CTC is $2,200 per child and is fully nonrefundable up to tax liability. The CDCC uses the enhanced rate schedule, which steps down from 35% by 1 point per $4,000 of joint AGI over $150,000, giving 32% × $6,000 = $1,920. The traditional IRA is not deductible because both spouses are covered by workplace plans, and the auto-loan interest deduction does not apply. Most wrong answers used one of these: TCJA-sunset rules (personal exemptions, $1,000 CTC with phaseout, 10/15/25 brackets); the old 20% CDCC rate; no $2,000 non-itemizer charitable deduction; a $2,000 CTC; or only $300–$400 per child of CTC treated as nonrefundable." +us,scenario_099,federal_income_tax_before_refundable_credits,47,llm_error,credit_phaseout,False,,"For 2026, the One Big Beautiful Bill Act (OBBBA) rules apply. The standard deduction is $32,200, and non-itemizers on a joint return get a $2,000 charitable deduction, which brings taxable income to $125,529.30. The CTC is $2,200 per child, and the CDCC rate drops from 35% by 1 point for each $4,000 of joint AGI over $150,000, so it is 32% here, giving $1,920 on $6,000 of expenses. The traditional IRA contributions are not deductible because of the active-participant phaseout, and the auto loan interest is not deductible. Most wrong answers used the old 20% CDCC rate, left out the $2,000 non-itemizer charitable deduction, used the pre-OBBBA $2,000 CTC, or applied the expired-TCJA rules (personal exemptions, $1,000 CTC)." us,scenario_099,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The trap is §24(d)'s liability-limitation prong: the refundable CTC is the *lesser* of the earned-income formula (15% of earnings over $2,500), the per-child refundable cap, and the portion of the CTC left unused after the credit offsets tax liability. This married-filing-jointly household has AGI near $159,676 (wages $165,000 plus $1,010 interest, $144 dividends, capital netting limited to a $3,000 loss, less $3,232 of traditional 401(k)/IRA contributions and the $338 educator expense) and roughly $127,000 of taxable income after the 2026 MFJ standard deduction, producing about $17,400 of federal income tax before credits — far more than the $4,400 of 2026 OBBBA CTC ($2,200 × 2), so the entire CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is zero because $159,676 is far beyond the two-child MFJ phase-out, and the CDCC is nonrefundable. Every wrong model treated a refundability cap or the earned-income formula as a standalone entitlement instead of applying the unused-credit ceiling." -us,scenario_099,free_school_meals_eligible,39,llm_error,state_local_rule,False,,"California runs universal free school meals (the CA Universal Meals Program, Education Code §49501.5, in effect since the 2022-23 school year). Every enrolled K-12 student gets free breakfast and lunch regardless of income. Because of this, PolicyEngine sets school_meal_tier to FREE even though the household's income is 5.03 times the poverty guideline and it has no categorical eligibility. The 9-year-old is in K-12, which produces $1,130.96 a year in free-meal support. Every model that answered applied only the federal NSLP 130%-of-FPL income test, so each one returned No by missing the state's universal program. Kimi-k3 gave no answer." +us,scenario_099,free_school_meals_eligible,40,llm_error,state_local_rule,False,,"California runs a universal school meals program (enacted by AB 130 in 2021 and in effect since the 2022-23 school year), so every enrolled K-12 student gets free breakfast and lunch at no charge, whatever the household's income. PolicyEngine sets school_meal_tier to FREE for California households even though this household's income is 5.03 times the poverty guideline and it has no categorical eligibility. The 9-year-old is in K-12, so the free meal support comes to $1,130.96 a year. Every model with reasoning applied only the federal National School Lunch Program test (130% of the poverty line, or categorical eligibility) and never applied California's universal program, which overrides that test." us,scenario_099,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_099,payroll_tax,41,llm_error,state_local_rule,False,,"Federal FICA was easy: 7.65% of the $165,000 gross wages gives $12,622.50, and there is no Additional Medicare Tax. Most models got that part right. The missing piece is California's mandatory employee SDI contribution. In 2026 it is 1.3% of all wages with no wage cap, which adds $2,145 and brings the total to $14,767.50. Wrong answers fall into four groups: models that left out SDI entirely, models that used the outdated 1.1% (2024) or 1.2% (2025) rate, models that subtracted the employer-sponsored insurance premiums from the wage base even though the reference taxes gross wages, and models whose final number did not match their own working." +us,scenario_099,payroll_tax,42,llm_error,thresholds_rates,False,,"The federal part is simple: 7.65% employee FICA on the full $165,000 of gross wages comes to $12,622.50, and there is no Additional Medicare Tax. The trap is California's mandatory employee SDI. For 2026 it is 1.3% with no wage cap, applied to the full gross wages, which adds $2,145. The wrong answers fall into four groups: some dropped SDI entirely, some used the outdated 1.1% (2024) or 1.2% (2025) rate, some subtracted employer-sponsored insurance premiums from the FICA and SDI wage base, and some submitted numbers their own arithmetic does not support." us,scenario_099,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -774,7 +774,7 @@ us,scenario_099,spouse_medicaid_eligible,1,parse_contract_failure,missing_output us,scenario_099,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_099,state_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's California itemized deduction reuses the federal charitable_deduction, and for 2026 that deduction already has the OBBBA 0.5%-of-AGI floor (IRC §170(b)(1)(I)) applied. California does not adopt that floor because its IRC conformity date under R&TC §17024.5 is earlier, so the CA charitable deduction is cut by $797.92, worth about $64 of tax.","The engine itemizes $20,476.57: real estate taxes of $8,653.50 plus charitable gifts of $11,823.08 after a 0.5%-of-AGI floor. It allows no medical deduction because out-of-pocket medical costs (at most $11,400) are below the 7.5% floor of about $11,969. That leaves $139,108.47 of taxable income, which the 2026 indexed MFJ brackets tax at $5,778.21; subtracting $1,284.46 of exemption credits (two personal, two dependent) gives $4,493.74. The wrong answers did one or more of these: took the standard deduction, created a medical deduction by counting the two premium fields twice, removed employer-sponsored insurance premiums from AGI, got the exemption credits wrong, or used unindexed or inflated brackets. The closest answers deducted charity without the 0.5% floor. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076)." +us,scenario_099,state_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The correct path starts from CA AGI of $160,066.80. That is wages net of the $3,088 traditional 401(k) deferrals, plus interest and dividends, less the $3,000 capital-loss cap. The traditional IRA deduction is fully phased out because both spouses are active plan participants, and the $337.50 educator deduction is added back. CA itemized deductions are only real estate taxes ($8,653.50, with no SALT cap) plus charity ($12,621), totaling $21,274.50. Out-of-pocket medical costs of $11,400 fall below the 7.5% floor, and the $1,263 of employee expenses falls below the 2% floor. This gives taxable income of $138,792.30, bracket tax of $5,871.48, and $1,256 of personal and dependent exemption credits. Most wrong answers went wrong in one of these ways: they used the standard deduction instead of itemizing, double-counted health premiums to create a medical deduction, excluded employer-paid ESI premiums from wages, misstated the 2026 bracket tax, or omitted or mis-sized the $1,256 exemption credits." us,scenario_099,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_100,child1_chip_eligible,6,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child not to qualify for Medicaid. At age 6 and this income, Child 1 qualifies for Montana Medicaid under the OLDER_CHILD category, so Medicaid eligibility categorically precludes CHIP eligibility; the CHIP income ceiling and existing employer-sponsored insurance do not alter that result." @@ -786,7 +786,7 @@ us,scenario_100,child2_early_head_start_eligible,1,llm_error,categorical_eligibi us,scenario_100,child2_head_start_eligible,4,llm_error,age_disability,False,,"Head Start for preschool-age children covers children from age 3 through compulsory school age, so a 5-year-old is squarely inside the band — it is the 6-year-old Child 1 who falls outside it, and Early Head Start that stops below age 3. On top of that, the household's TANF receipt of $505.41 monthly confers categorical eligibility on all members, and its $1,875.40 adjusted gross income is far below the poverty guideline for three people, so income eligibility holds independently. Three models cut Head Start off at age 4 and one defaulted the flag to zero with no derivation; all four therefore never applied the categorical TANF pathway or the income test that both resolve to eligible." us,scenario_100,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Montana Medicaid applies a young-child eligibility pathway to the 5-year-old dependent. The child's household MAGI is 0.07 times FPL, below the applicable young-child income threshold, so the eligibility test returns Yes." us,scenario_100,child2_wic_eligible,12,llm_error,age_disability,False,,"WIC's child category covers children from age 1 until their 5th birthday, so the age test is strictly under 5. Child 2 is age 5, has passed that birthday, and is not in the child category. They are also not an infant, pregnant, postpartum, or breastfeeding, so they fall into no WIC category. The household's very low income doesn't matter once the categorical age test fails. Every model that answered treated age 5 as within the WIC child range, and none of them applied the strict under-5 cutoff." -us,scenario_100,federal_refundable_credits,36,reference_engine_defect,credit_phaseout,False,"PolicyEngine computes EITC and ACTC earned income from gross employment_income ($5,915) without subtracting the $3,859 traditional 401(k) elective deferral. IRC §32(c)(2)(A)(i), as amended by EGTRRA 2001 and incorporated into §24(d), counts employee compensation only ""if such amounts are includible in gross income."" IRS Pub 596 likewise says employee pay counts as earned income only if it is taxable. That gives $2,056 of earned income, an EITC of 40% × $2,056 = $822.40 and an ACTC of $0, not $2,878.10.","The reference applies the 40% two-child EITC phase-in rate to the full $5,915 gross wages, which gives $2,365.89. It adds an ACTC of 15% × ($5,915 − $2,500), which gives $512.21. The main split is over the earned-income base: eight models subtracted the $3,859 traditional 401(k) deferral and got $822.40 with no ACTC. The other wrong answers came from model-specific errors: using the expired $3,000 ACTC threshold, multiplying the earnings-based ACTC limit per child, counting the $710 overtime premium twice, treating ESI premiums as pre-tax wage reductions, ignoring the earnings limit on the refundable CTC, or submitting numbers that contradict the model's own work. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." +us,scenario_100,federal_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Earned income for both the EITC (IRC §32(c)(2)) and the ACTC (§24(d)) counts only compensation includible in gross income. The $3,859 of traditional 401(k) elective deferrals are excluded, so earned income is $5,914.72 − $3,859 = $2,055.72. That puts the EITC in the 40% phase-in at $822.29. It also puts earned income under the $2,500 ACTC floor, so refundable CTC is $0. Most models instead used the full $5,915 of gross wages and got $2,366 EITC plus $512.25 ACTC = $2,878.25. Others took out deductions that don't apply (IRA, health premiums, ESI), added overtime a second time, invented refundability rules, or returned 0 with no basis." us,scenario_100,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Free school meals are awarded through two independent pathways, and this Montana household clears both: it receives $8,625.89 of SNAP, which categorically confers free-meal status, and its countable income of $11,979.68 ($5,914.72 wages plus $6,064.96 TANF) is 44% of the federal poverty guideline for a three-person household, far under the 130% free-meal ceiling. The two children, ages 5 and 6, are both school-age, so the free tier pays a $7.15 daily per-child subsidy for the contiguous-US school year, $2,261.92 annually. The trap is that SNAP and TANF receipt are derived outputs of this same household, not listed input facts, so a model that only credits explicitly stated benefit receipt loses the categorical pathway — but even the wage-only figure of $5,915 sits at roughly 22% of the poverty guideline, so the income test alone still lands in the free tier. Both models returned 0 without running either test." us,scenario_100,head_medicaid_eligible,4,llm_error,thresholds_rates,False,,"The head is a 46-year-old parent of two young children with MAGI of about $5,915, which is 0.07 × FPL for a household of three. That is far below Montana's Medicaid limit for the parent/caretaker category, so the head is eligible. MAGI-based Medicaid has no asset test, and having employer-sponsored insurance does not bar eligibility. The wrong models used an invented FPL dollar figure, applied an asset test, or treated ESI as disqualifying. None of them compared 7% of FPL against the parent income limit." us,scenario_100,head_medicare_eligible,2,llm_error,categorical_eligibility,False,,"Medicare eligibility below age 65 does not follow from a current disability flag alone; it requires a qualifying Medicare pathway, such as entitlement after the applicable Social Security disability waiting period or eligibility based on end-stage renal disease. The 46-year-old head has no listed Medicare-qualifying entitlement or condition, so the disability input does not change the result from ineligible." @@ -794,11 +794,11 @@ us,scenario_100,head_wic_eligible,4,llm_error,categorical_eligibility,False,,"WI us,scenario_100,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_100,payroll_tax,7,llm_error,payroll_tax_base,False,,"The trap is the payroll-tax wage base: PolicyEngine assesses employee OASDI at 6.2% and HI at 1.45% on the full $5,915 of employment income, giving $366.71 + $85.76 = $452.48. None of the listed health inputs touch that base — the $20,644 employer-sponsored insurance premium and the two $564 individually paid premiums are health-coverage and medical-expense inputs, not Section 125 salary reductions — and the $710 FLSA overtime premium is already inside the annual gross wage figure rather than additional to it. Traditional 401(k) and IRA amounts also leave FICA wages untouched, and Montana imposes no mandatory employee-side state payroll tax, so the answer is exactly 7.65% of wages. The wrong answers split into base errors (subtracting premiums to $4,787 or to $0, or adding the overtime premium to reach $6,625) and, for the three models that found the right base, an arithmetic slip, an invented rounding, and a value/explanation mismatch." us,scenario_100,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier. This household is at 44% of the federal poverty guideline and also meets categorical eligibility, so its school_meal_tier is FREE and reduced_price_school_meals_eligible is 0." -us,scenario_100,snap,47,prompt_ambiguity,age_disability,False,,"SNAP gross income is $998.31 a month: $492.92 in wages plus about $505.39 in Montana TANF cash, which counts as unearned income. The deductions are the $209 standard deduction, the $98.58 earned income deduction (20%), and a $453.64 excess shelter deduction. The shelter deduction comes from Montana's standard utility allowance, which counts as a shelter cost even though no rent is listed. There is no excess medical deduction because the head receives no disability benefit such as SSI or Social Security disability, so the head does not count as disabled for SNAP. Net income is $237.09, and 30% of that ($71.10) is subtracted from the maximum allotment. That gives $713.90 a month for January–September and $733.60 for October–December, when the FY2027 allotment of $802.60 applies. Most models used wages only, left out TANF, added a medical deduction, and landed near the maximum allotment ($9,000–$9,420). The models that counted TANF left out the shelter deduction and landed around $7,000–$7,800. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification." +us,scenario_100,snap,48,prompt_ambiguity,age_disability,False,,"PolicyEngine's SNAP gross income is $998.31/month. That is $492.92 in wages plus about $505/month in non-wage cash-benefit income. The frozen reference subtracts the $209 standard deduction, the $98.58 earned income deduction and a $453.64 excess shelter deduction, with no medical deduction, giving $237.09 net and a $71.10 contribution. Its approximately $713.90 January–September and $733.60 October–December monthly amounts aggregate to the frozen annual value of $8,625.89; the latter uses the engine's projected FY2027 $802.60 maximum. The publication convention c_snap_hold_fy2026 holds the FY2026 $785 maximum and schedule all year; with the rounding fixes, the earlier engine gave $713/month ($8,556/year). The $453.64 shelter deduction comes from r30_snap_heat_and_eat_sua. Correcting that defect on the record's stated-facts reading takes no utility-allowance shelter deduction and gives $691 net, a $208 contribution and $577/month ($6,924/year), now reproduced by policyengine-us 2.41.0. Many models counted wages only and added a medical deduction, landing near $9,000–9,420. Taking no shelter deduction agrees with the recorded correction; the TANF-counting models came in above its $6,924, although most were below the frozen reference. Neither disability reading is established, so the output remains excluded. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, since fixed upstream by PolicyEngine/policyengine-us#10033, which the policyengine-us 2.41.0 references include) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification." us,scenario_100,ssi,9,llm_error,categorical_eligibility,False,,"The generic `is disabled` household fact does not make the head categorically eligible for SSI under PolicyEngine; the controlling SSI-specific indicator is `is_ssi_aged_blind_disabled=False`. All three individuals fail the aged-65-or-older, blind, or qualifying-disabled requirement, so income exclusions, resources, and benefit-rate calculations are never reached and household SSI is $0." -us,scenario_100,state_refundable_credits,45,reference_engine_defect,credit_phaseout,False,,"Montana's state refundable credits for this household come only from the Montana EITC. For 2026 that credit is refundable and equals 20% of the federal EITC. The federal EITC is the 40% two-child phase-in rate applied to the head's full $5,915 of earned income, which gives $2,365.89, so the Montana credit is $473.18. Most models went wrong in one of three ways: they said Montana has no state EITC, no income tax, or only a nonrefundable EITC (giving $0); they used the older 3% or 10% rate; or they cut earned income by the $3,859 traditional 401(k) deferral, which gives a federal EITC of $822.40. A few also added a $1,200 young-child credit that PolicyEngine does not count in this household's Montana refundable credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." -us,scenario_100,tanf,47,llm_error,categorical_eligibility,False,,"The family of three qualifies for Montana TANF. Its $2,800 in bank assets is under the $3,000 resource limit, and the one vehicle is excluded. The head gets no SSI because $2,800 is over the $2,000 SSI resource limit, so there is no unearned income to count. Countable earnings are 0.75 × ($5,915/12 − $200) = $219.69 a month, after the $200 disregard plus 25% of the rest. That amount is subtracted from Montana's 2026 three-person payment standard of about $725.10 a month, which gives about $505.41 a month, or $6,064.96 a year. Most wrong answers either returned $0 by inventing reasons the family was ineligible (no modeling, asset limits that are too low, SSI receipt, or counting overtime twice) or used an outdated payment standard around $588 or a 50% earnings disregard." -us,scenario_101,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"The correct path is: AGI of $101,130 ($103,000 wages minus the $1,870 capital loss; the pension and veterans benefits are not taxable), minus the $16,100 2026 single standard deduction and nothing else, gives taxable income of $85,030. Under the 2026 brackets that OBBBA made permanent (10% to $12,400, 12% to $50,400, then 22%), that is $1,240 + $4,560 + $7,618.60 = $13,418.60, with no nonrefundable credits. The wrong answers fall into three groups. One group applied the TCJA-sunset rules ($8,300 standard deduction plus a $5,300 personal exemption, with 15%/25% brackets). A second group used 2025 or guessed standard deductions and bracket thresholds. A third group subtracted the $1,400 auto loan interest even though nothing in the facts shows a qualified passenger vehicle loan. Several models reached $13,418.60 or $85,030 and then reported a different number." +us,scenario_100,state_refundable_credits,46,llm_error,thresholds_rates,False,,"For 2026, the Montana EITC is refundable and equals 20% of the federal EITC. Federal EITC earned income is wages included in gross income, so the $3,859 traditional 401(k) deferral comes out of the $5,914.72 in wages, leaving $2,055.72. At the 40% phase-in rate for two children that gives $822.29, and 20% of that is $164.46. Most models fell into one of three traps: they used an outdated Montana match rate (3%, 10% or 12%), they applied 40% to gross wages ($2,366) without removing the 401(k) deferral, or they said Montana has no refundable EITC." +us,scenario_100,tanf,48,llm_error,thresholds_rates,False,,"The household is eligible for Montana TANF. Bank assets of $2,800 are under the $3,000 resource limit and the vehicle is excluded. Take-up is assumed. SSI is $0 because $2,800 is above the $2,000 SSI resource limit, so there is no SSI income to count. Monthly wages of $492.92 get the $200 + 25% disregard, leaving $219.69 of countable earnings. PolicyEngine subtracts that from its 2026 three-person payment standard of about $725/month, giving about $505.41/month, or $6,064.96 a year. Most wrong answers used an outdated $588 (or other too-low) payment standard, used a 50% disregard, or wrongly called the household ineligible because of assets, SSI, missing receipt, or the program not being modeled." +us,scenario_101,federal_income_tax_before_refundable_credits,41,llm_error,thresholds_rates,False,,"The reference is simple 2026 OBBBA-law arithmetic. AGI is $101,130 ($103,000 in wages minus the $1,870 capital loss). Subtracting the $16,100 standard deduction leaves $85,030 of taxable income, and the 2026 single brackets (10% to $12,400, 12% to $50,400, then 22%) give $1,240 + $4,560 + $7,618.60 = $13,418.60. The models went wrong in three main ways. One group assumed the TCJA would sunset and used a personal exemption with 15%/25% brackets. A second group used the wrong standard deduction or bracket thresholds. A third group subtracted the $1,400 auto loan interest as an OBBBA qualified vehicle loan interest deduction, even though nothing says the loan is for a new, U.S.-assembled vehicle. Several models also worked out the correct $13,418.60 and then submitted a different number." us,scenario_101,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_101,payroll_tax,8,llm_error,other,False,,"The correct calculation is simple. Employee Social Security is 6.2% of the full $103,000 of wages, or $6,386, because the wages are below the 2026 wage base. Employee Medicare is 1.45%, or $1,493.50. No Additional Medicare Tax applies below $200,000, and Texas has no employee state payroll tax, so the total is $7,879.50. Most models reached exactly $7,879.50 in their reasoning and then submitted a different number with a made-up ""adjustment."" One model treated the listed health insurance premiums as a pre-tax payroll exclusion, another submitted zero, and one gave no output." us,scenario_101,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident has $0 of state income tax before refundable credits regardless of the listed taxable-income deductions or credits. Taxes from other jurisdictions do not apply because the household resides in Texas." @@ -831,9 +831,9 @@ us,scenario_108,head_medicare_eligible,1,parse_contract_failure,missing_output,F us,scenario_108,payroll_tax,4,llm_error,payroll_tax_base,False,,"Only the head's $7,656 of wages is subject to employee FICA. Social Security benefits and survivor benefits are not wages. The correct figure is 6.2% × $7,656 = $474.67 plus 1.45% × $7,656 = $111.01, for $585.68 in total. Wisconsin has no mandatory employee state payroll tax, and Additional Medicare Tax does not apply. Every wrong answer comes from an arithmetic slip, a submitted value that doesn't match the model's own correct calculation, or a missing output. None reflects a different reading of the rules." us,scenario_108,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_108,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_108,snap,44,llm_error,categorical_eligibility,False,,"Wisconsin FoodShare uses broad-based categorical eligibility (BBCE) through TANF-funded non-cash benefits, with a 200% FPL gross limit. Monthly gross income of $2,581.83 is under that limit (about $2,608), and assets of $270 also pass, so the household is categorically eligible. That waives the 100% FPL net income test. Net income of $2,036 gives a 30% contribution of $611, which is more than the $298 one-person maximum, so the formula benefit is zero. But categorically eligible one- and two-person households still get the minimum allotment, which is $24 a month in FY2026 (8% of $298), or $288 a year. Most models applied the net income test or treated a zero formula benefit as $0. The two $276 answers used the outdated $23 minimum, and several other answers were unsupported benefit amounts." -us,scenario_108,state_income_tax_before_refundable_credits,9,llm_error,state_local_rule,False,,"Every model that answered computed a small positive Wisconsin gross tax, about $110-$165, on roughly $18,048 of Wisconsin income (wages plus the taxable non-Social Security survivor benefits) after the sliding-scale standard deduction and exemptions. None of them then brought that tax to zero with Wisconsin's nonrefundable school property tax credit for renters. That credit is 12% of rent treated as property tax (20% of rent if heat is included, 25% if not, capped at $2,500). On $7,200 of rent it comes to $172.80-$216. That is more than any model's gross tax, so Wisconsin tax before refundable credits is $0. The one model that mentioned the renter credit made it too small and left a balance." -us,scenario_108,state_refundable_credits,47,prompt_ambiguity,state_local_rule,False,,"The whole $271.39 is the refundable Wisconsin Homestead Credit. Household income is $20,590, which is $7,656 in wages plus $12,934 in Social Security survivor benefits; the separate $10,392 in survivor benefits is not part of it. Property taxes are 20% of the $7,200 rent, or $1,440, so the credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39. Most models added the $10,392 to reach $30,982, put the household over the $24,680 income limit and answered $0. Others never applied the Homestead Credit, which is refundable even when no tax is owed. A few found a partial credit but used the wrong income base, the wrong rent percentage or the wrong formula parameters. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)." +us,scenario_108,snap,45,llm_error,categorical_eligibility,False,,"Wisconsin's FoodShare uses broad-based categorical eligibility (BBCE) through TANF-funded non-cash benefits, with a 200% FPL gross income limit. The household's gross income of $2,581.83/month is just under that limit ($2,608.33), so the household is categorically eligible and does not face the 100% FPL net income test. Its benefit formula still comes out negative: the 30% expected contribution of about $611 exceeds the $298 maximum allotment. However, categorically eligible one- and two-person households receive the minimum allotment, which is 8% of the one-person maximum. For FY2026 that is $24/month, or $288 for the year. Most models applied the net income test, concluded the household was ineligible, and answered $0. The others either used the outdated $23 minimum or invented a larger benefit." +us,scenario_108,state_income_tax_before_refundable_credits,10,llm_error,taxable_income_or_deductions,False,,"Every substantive answer started from a Wisconsin income base of $18,048, which counts the $10,392 non-Social-Security survivor benefits as taxable. The engine's income base is $20,590, which is the $7,656 in wages plus the $12,934 in Social Security survivor benefits, and Wisconsin exempts Social Security entirely. That leaves only the $7,656 in wages, which falls entirely under Wisconsin's standard deduction (about $13,000–$14,000 at this income) plus the $700 personal exemption and the $250 age-65+ exemption. Even on the models' own $18,048 base, Wisconsin's nonrefundable school property tax credit for renters applies. It treats 20–25% of the $7,200 rent as property tax and credits 12% of that, giving roughly $173–$216. That is more than every model's gross tax of $94–$162, so tax before refundable credits is $0 either way." +us,scenario_108,state_refundable_credits,48,prompt_ambiguity,state_local_rule,False,,"The whole state_refundable_credits amount is the Wisconsin Homestead Credit. PolicyEngine's wi_homestead_income counts only wages of $7,656 and Social Security of $12,934, for $20,590. The separate $10,392 survivor-benefits input is not part of it, so the household sits under the $24,680 income ceiling. Rent of $7,200 gives property taxes accrued at 20%, or $1,440, and the credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39. Most models added the $10,392 to reach $30,982 and wrongly ruled the household ineligible. Others ignored the refundable homestead credit entirely, or used the wrong rent percentage, threshold or rate. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)." us,scenario_109,child1_chip_eligible,6,llm_error,health_coverage,False,,"CHIP is limited to children who do not qualify for Medicaid. At approximately $30,000 of household income, the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility even though the income is below CHIP's upper limit." us,scenario_109,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Florida's refusal of adult Medicaid expansion does not touch children's coverage: the state's mandatory MAGI pathway for the older-child category (ages 6-18) runs to 133% FPL, and a 9-year-old in a household of five with roughly $21,600 of MAGI sits at 0.72 FPL — barely half the threshold. The only computation required is household MAGI from the $30,000 of self-employment income divided by the family-of-five poverty guideline, then compared with 133% FPL; no asset test, no enrollment fact, and no SSI linkage enters the children's MAGI pathway. Both models returned 0 without ever running that ratio, treating Medicaid eligibility as something the prompt had to assert rather than something the stated income and age determine." us,scenario_109,child2_chip_eligible,6,llm_error,health_coverage,False,,"CHIP eligibility requires the child to be ineligible for Medicaid, not merely under age 19 and below CHIP's upper income limit. At age 6 and this household income, child2 qualifies for Medicaid under Florida's OLDER_CHILD category, so the Medicaid–CHIP mutual-exclusivity rule makes child2 ineligible for CHIP." @@ -846,20 +846,20 @@ us,scenario_109,child3_head_start_eligible,1,llm_error,categorical_eligibility,F us,scenario_109,child3_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The dispositive rule is Florida's age-based INFANT Medicaid category, which covers children under age 1 at MAGI up to 185% FPL — a far higher threshold than the adult standards, and one that applies in a non-expansion state. The household's MAGI is 0.72 FPL, so the only two conditions that matter, age under 1 and income under the infant threshold, are both satisfied. All three models denied eligibility by routing the infant through adult or generic standards: one gave no derivation, one required an SSI/disability-style marker that MAGI categories do not use, and one computed the income test correctly and then overrode it with a nonexistent asset test plus a Medicaid-expansion and enrollment requirement." us,scenario_109,child3_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is derived from the child's age and household income; it does not require the prompt to explicitly state that the child is WIC-eligible. Child 3 is under age five, and the household's approximately $30,000 annual income is below the 2026 WIC limit of 185% of the federal poverty guideline for a five-person household." us,scenario_109,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The 2026 married-filing-jointly standard deduction does not reduce this household's taxable income to zero after the deductible half of self-employment tax. The resulting pre-credit income tax is fully offset by $2,792.91 of the $6,600 Child Tax Credit allocated as nonrefundable, and tax after nonrefundable credits is floored at zero rather than becoming negative." -us,scenario_109,federal_refundable_credits,41,llm_error,thresholds_rates,False,,"The reference is the full 2026 three-child EITC maximum of $8,231 plus an ACTC of $3,807.09. Earned income is $30,000 minus the deductible half of SE tax ($2,119.43), or $27,880.57. That is on the EITC plateau and below the MFJ phase-out start. The ACTC is 15% × ($27,880.57 − $2,500), which is below the 3 × $1,700 refundable cap under the post-OBBBA CTC. The wrong answers made one or more of four mistakes. Some used the expired-TCJA $1,000-per-child CTC with a $3,000 threshold. Some measured earned income as gross $30,000 or as net SE earnings of $27,705 instead of SE profit minus half of SE tax. Some phased out the EITC when it should not have been. Others used a wrong 2026 EITC maximum." +us,scenario_109,federal_refundable_credits,42,llm_error,thresholds_rates,False,,"The reference has two parts. The first is the full 2026 three-child EITC maximum of $8,231, because earned income of about $27,881 sits on the plateau, below the married-filing-jointly phase-out start. The second is the Additional CTC: 15% × (earned income − $2,500), where earned income for self-employment is $30,000 minus the deductible half of SE tax ($2,119.43). That gives 15% × $25,380.57 = $3,807.09, under the post-OBBBA cap of $1,700 per child ($5,100). Most wrong answers made one of three mistakes: they assumed TCJA had expired and used a $1,000-per-child ACTC ($3,000), they applied the 15% formula to gross $30,000 ($4,125) or to $27,705 ($3,781) instead of $27,880.57, or they used an EITC maximum other than $8,231. A smaller group wrongly phased out the EITC, dropped the ACTC, or reported zero credits." us,scenario_109,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The FREE tier here is over-determined: the household's $8,020.55 SNAP benefit confers categorical free-meal eligibility outright, and independently $30,000 for a five-person household is 78% of the federal poverty guideline, far under the 130% free-meal cutoff. The only fact requiring inference is school attendance — never stated explicitly — which follows from the children aged 9 and 6 being K-12 students, generating $7.15 per child per day net of the $0.87 paid portion for $2,261.92 annually. The trap is the instruction to treat unlisted statuses as false: a model that applies it to school enrollment erases both students and skips both qualifying tests. Both models defaulted to 0 on that reading rather than deriving the students from age and then applying either the SNAP categorical pathway or the 130% FPG income test." us,scenario_109,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Low MAGI alone does not establish adult Medicaid eligibility in Florida. The age-30 head is not a dependent, receives no SSI, and fits none of Florida's active Medicaid eligibility categories, so the category is NONE despite MAGI income of 0.72 times FPL." us,scenario_109,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is individual, not household-wide: only a pregnant, postpartum, or breastfeeding woman, an infant, or a child under age five can qualify. The infant satisfies that requirement, but the 30-year-old Head has no listed pregnancy, postpartum, or breastfeeding status, so the Head remains ineligible despite the household meeting the income test." us,scenario_109,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price and free school meals are mutually exclusive output tiers in this calculation. The household’s 0.78 school-meal FPG ratio and categorical eligibility place it in the superior FREE tier, so it receives positive free-meal support and no reduced-price support." us,scenario_109,self_employment_tax,6,llm_error,other,False,,"The Schedule SE computation here is fully determined: net earnings from self-employment are 92.35% of $30,000 = $27,705, entirely below the Social Security wage base, so the tax is $27,705 × 15.3% = $4,238.865, which PolicyEngine reports as $4,238.86. Every model named the correct base fraction, the correct 15.3% combined rate, and correctly noted the wage base is not binding — none missed a rule. The separation is purely in executing the product: each model reported a whole-dollar figure that is $2 to $6 away from $4,238.87, whereas correct rounding of the true product to whole dollars gives $4,239. The trap is arithmetic precision on 0.9235 × 0.153 × $30,000, not any element of the SE tax rule." -us,scenario_109,snap,42,llm_error,thresholds_rates,False,,"The correct SNAP calculation counts all $2,500 of monthly self-employment receipts, with no SE-tax reduction and no business costs listed. It subtracts the 20% earned income deduction ($500) and the FY2026 standard deduction for five people ($261), giving $1,739. The $800 rent is below half of that amount ($869.50), and no utility costs are listed, so there is no excess shelter deduction. Thirty percent of net income ($521.70) is rounded up to $522 and subtracted from the FY2026 five-person maximum of $1,183, giving $661 a month, or $7,932 a year. Most wrong answers came from using FY2025 or guessed figures for the standard deduction and maximum benefit ($254/$1,155 and others), skipping the round-up to a whole dollar, adding a shelter deduction or utility allowance that doesn't apply, or miscounting the household or eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))." +us,scenario_109,snap,43,llm_error,thresholds_rates,False,,"This is a simple SNAP case. Gross income is $2,500 a month. Subtract the 20% earned income deduction ($500) and the FY2026 standard deduction for five people ($261) to get net income of $1,739. Rent of $800 is below half of adjusted income ($869.50), so there is no excess shelter deduction, and no utility allowance applies. The expected contribution, 30% of net income rounded up, is $522. Subtracting it from the FY2026 maximum allotment for five ($1,183) gives $661 a month, or $7,932 a year. Most wrong answers came from using stale or guessed values for the standard deduction or maximum allotment. Others applied a shelter or SE-tax deduction that does not exist here. The rest used the right parameters but annualized the unrounded $661.30 instead of the whole-dollar $661 monthly benefit. A few models wrongly denied eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))." us,scenario_109,spouse_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Florida’s non-expansion Medicaid program does not provide a general low-income pathway for a nondisabled 29-year-old adult. The spouse qualifies through no categorical pathway: household MAGI exceeds Florida’s parent/caretaker limit, and the presence of an infant does not establish pregnancy or postpartum status." us,scenario_109,spouse_wic_eligible,8,llm_error,categorical_eligibility,False,,"WIC eligibility for an adult requires the adult herself to be pregnant, breastfeeding, or within the applicable postpartum period; merely living with an infant does not establish any of those statuses. The prompt makes every unlisted status false, so the spouse is not pregnant, postpartum, or breastfeeding and fails categorical eligibility regardless of household income." -us,scenario_110,federal_income_tax_before_refundable_credits,45,llm_error,taxable_income_or_deductions,False,,"Under OBBBA's permanent 2026 rules, a single filer gets the $16,100 standard deduction plus the new non-itemizer charitable cash deduction (here $387). That leaves taxable income of $153,455.47. Of this, $112,571.70 is taxed at ordinary rates under the 2026 brackets (10%/12,400, 12%/50,400, 22%/105,700, 24%), giving $19,615.21, and $40,883.77 of qualified dividends and LTCG is taxed at 15%, giving $6,132.56. The largest cluster of answers ($25,841) got every step right except the $387 charitable deduction. Most other wrong answers assumed the TCJA had expired, with personal exemptions, 2%-floor miscellaneous deductions and the old brackets. Others excluded employer-paid premiums from wages, invented an overtime deduction from the hours worked, allowed a traditional IRA deduction that phases out at this income, or used outdated brackets and standard-deduction amounts." +us,scenario_110,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The reference uses permanent post-OBBBA 2026 law. AGI is $169,943: wages minus only the $4,245 traditional 401(k), with no IRA deduction (phased out for an active plan participant), no exclusion for ESI premiums and no overtime deduction. Deductions are the $16,100 single standard deduction plus the new 2026 non-itemizer deduction for the $387 of cash gifts, giving taxable income of $153,455. Ordinary tax on $112,572 is $19,615 under the $12,400/$50,400/$105,700 brackets, and the $40,884 of qualified dividends and LTCG is all taxed at 15% ($6,133). The cluster of models at $25,841 did everything else right but missed the $387 non-itemizer charitable deduction (about $93 at the 24% rate). Other models assumed the TCJA would sunset (personal exemption, 2% miscellaneous deductions), excluded ESI premiums from wages, invented an overtime deduction, or used outdated brackets." us,scenario_110,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_110,payroll_tax,16,llm_error,payroll_tax_base,False,,"The correct employee payroll tax applies 6.2% Social Security tax ($6,200) and 1.45% Medicare tax ($1,450) to the full $100,000 of listed gross wages. That gives $7,650, with no Additional Medicare Tax because wages are well below $200,000, and Ohio has no employee state payroll tax. The main mistake was shrinking the FICA wage base. Most models subtracted the $6,889 employer-sponsored insurance premium as if it were a Section 125 salary reduction, but the facts never say it is. One model subtracted the traditional 401(k) deferral, which does not reduce FICA wages. Other models applied the Additional Medicare Tax to non-wage income, or submitted numbers that contradicted their own correct $7,650 calculation." +us,scenario_110,payroll_tax,17,llm_error,payroll_tax_base,False,,"The reference applies employee FICA to the full $100,000 of stated gross wages: 6.2% Social Security ($6,200, well under the wage base) plus 1.45% Medicare ($1,450). That totals $7,650, and there is no Additional Medicare Tax because wages are below $200,000. The household facts never say the $6,889 employer-sponsored insurance premium is a Section 125 salary reduction, and the prompt says not to infer unlisted facts. The largest group of models still subtracted it from FICA wages and reported $7,123. The other wrong answers came from excluding the 401(k) deferral from FICA wages, applying Additional Medicare Tax to non-wage income, or submitting a value that did not match the model's own reasoning." us,scenario_110,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_110,state_income_tax_before_refundable_credits,47,llm_error,thresholds_rates,False,,"Ohio AGI equals federal AGI of $169,942.94. That figure counts wages net only of the traditional 401(k) deferral. The $198 IRA deduction is fully phased out because the head has workplace coverage and high income, and employer insurance premiums are not subtracted. The $1,900 personal exemption for income above $80k leaves $168,042.94 of taxable income. The 2026 Ohio schedule taxes income above $26,050 at a flat 2.75% plus a fixed $332 base: $332 + 0.0275 × $141,992.94 = $4,236.81. No nonrefundable credit applies. Most models either left out the $332 base (landing near $3,905), kept the repealed 3.5% bracket above $100,000 (landing near $4,400–$4,500), or changed AGI by deducting the phased-out IRA, subtracting the $6,889 employer insurance premiums, adding back the $954 muni interest, or skipping or misstating the $1,900 exemption." +us,scenario_110,state_income_tax_before_refundable_credits,48,llm_error,thresholds_rates,False,,"Federal and Ohio AGI is $169,943. That is $100,000 in wages minus the $4,245 traditional 401(k) contribution, plus $74,188 of interest, dividends, capital gains and estate income. The $198 traditional IRA deduction is fully phased out because the head is an active plan participant with high income, and the $6,889 of employer insurance premiums does not reduce AGI. After the $1,900 personal exemption that applies above $80,000 of Ohio AGI, Ohio taxable income is $168,043, and the 2026 schedule gives $332 + 2.75% × ($168,043 − $26,050) = $4,236.81. The most common miss was dropping the $332 base amount, which produces $3,905. Other models used the 3.5% bracket above $100,000, which no longer applies, the old $360.69 base, or other old brackets. Some also skipped or misstated the exemption, or got AGI wrong by deducting employer insurance premiums or the IRA contribution, or by adding back tax-exempt interest." us,scenario_110,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_111,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The IRA distribution produces $18,240 of adjusted gross income, while the Social Security retirement benefit is not taxable under the joint-return provisional-income thresholds. The applicable standard deduction eliminates taxable income, so the tax computation yields $0; both models expressly derived zero taxable income and zero liability but then submitted unsupported positive amounts." us,scenario_111,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"At 72, the head is outside the ACA expansion-adult category, which covers ages 19–64 up to 138% FPL. Their MAGI is also about 202% FPL, so that pathway fails twice. The head gets no SSI, so the SSI-recipient route is closed. The aged/blind/disabled route has an income limit near the SSI/100% FPL level, roughly $1,000–$1,300 a month. The head's unearned income is about $3,637 a month, so it is far over that limit even after the $20 general disregard. All four models treated being aged, blind or disabled plus low assets as enough to qualify and skipped the income test, which leaves medicaid_category = NONE." @@ -873,17 +873,17 @@ us,scenario_112,head_medicaid_eligible,5,llm_error,categorical_eligibility,False us,scenario_112,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires both financial eligibility and membership in a qualifying demographic category: pregnant, postpartum, or breastfeeding women, infants, or children under age five. The 31-year-old head has no listed qualifying status, and unlisted statuses are false, so passing the income test does not establish WIC eligibility." us,scenario_112,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employer-sponsored insurance premiums are not automatically employee pre-tax cafeteria-plan deductions from FICA wages; the listed $5,923 remains subject to employee Social Security and Medicare taxes. PolicyEngine computes $367.22 of Social Security tax and $85.88 of Medicare tax, totaling $453.10, with no Additional Medicare Tax or mandatory Texas employee payroll tax." us,scenario_112,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to positive net earnings from self-employment, after determining the applicable self-employment tax base. The household’s relevant net income is approximately -$2,101, so there are no positive net self-employment earnings and the tax base is zero; treating the $1,920 farm rent entry as a standalone positive self-employment tax base produces the wrong liability." -us,scenario_112,snap,47,prompt_ambiguity,categorical_eligibility,False,,"PolicyEngine counts wages ($493.58/mo) and the $13,000 financial assistance ($1,083.33/mo) but not farm rent. That gives gross income of $1,576.90 (121% FPL, inside Texas's 165% FPL BBCE limit). After the 20% earned-income deduction and the $209 standard deduction, net income is $1,269.19 (97% FPL). The household is categorically eligible through TANF non-cash BBCE. Thirty percent of net income ($380.76) exceeds the $298 maximum allotment, so the formula benefit is zero, but an eligible one-person household still gets the minimum allotment of about $24/month, or $287.68 for the year. Most models zeroed the benefit in one of three ways: they applied a 130% FPL gross test with no BBCE, they counted farm rent and failed the net test, or they let the formula reach $0 without applying the minimum benefit. Five models made the opposite error: they left out the financial assistance, or used a maximum allotment, and overstated the benefit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)." -us,scenario_114,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"AGI is about $106,573 for nearly every model. The answers split on the deduction stack. The reference itemizes $24,787.23: charity after the 0.5%-of-AGI floor, SALT of real estate tax plus about $5,367 of imputed Virginia income tax, and medical expenses above 7.5% of AGI that include a $2,434.80 Medicare Part B premium but exclude over-the-counter costs. It also takes the OBBBA senior deduction phased down to $4,105.65, which itemizers also get, and the $4,872 QBI deduction. Taxable income of $72,807 is then taxed at the permanent 2026 OBBBA single brackets (10/12/22%). Models that applied the TCJA sunset (personal exemption, 10/15/25% brackets, no QBI deduction), left out the senior deduction, understated SALT or medical, or took the standard deduction instead of itemizing all ended up with too much taxable income. A few models also imputed self-employment tax, dropped income, or made arbitrary final adjustments. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)." +us,scenario_112,snap,48,prompt_ambiguity,categorical_eligibility,False,,"PolicyEngine's SNAP countable income includes the wages and the $13,000 financial assistance but not the $1,920 farm rent. That makes gross income $1,576.90/mo (121% FPL), and net income after the 20% earned-income deduction ($98.72) and the $209 standard deduction is $1,269.19 (97% FPL). The household passes both income tests and is also categorically eligible through Texas's TANF non-cash BBCE. Because 30% of net income ($380.76) is more than the $298 maximum allotment, this one-person household gets the minimum allotment: 8% of the one-person maximum, or $23.84/mo ($24.37 in the FY2027 months), for $287.68 a year. Most models added farm rent to income and then failed the 130% gross or 100% net test, or they reduced the benefit to $0 without applying the minimum allotment. A few left out the financial assistance and computed a large benefit instead. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)." +us,scenario_114,federal_income_tax_before_refundable_credits,48,prompt_ambiguity,taxable_income_or_deductions,False,,"The frozen reference applies 2026 law under OBBBA, retaining the TCJA rates, QBI deduction and no personal exemption, using single brackets of 10% to $12,400, 12% to $50,400 and 22% on the remaining taxable income. The engine itemizes $24,787: $10,914 of charity after the 0.5%-of-AGI floor, $9,432 of SALT including its $5,367.30 Virginia withholding estimate, and $4,442 of medical including a $2,434.80 Medicare Part B premium it imputes and excluding the $2,000 of over-the-counter costs. Neither state income tax paid or withheld nor Medicare enrollment or a Part B premium is listed in the prompt. The phased-down $4,106 senior deduction and $4,872 QBI deduction leave the reference's taxable income at $72,807 and tax at $10,729.61. Some models applied pre-TCJA sunset law or omitted the senior deduction; others used different readings of the unlisted tax and Medicare inputs, including answers matching the engine's alternatives; some submitted figures unsupported by their own calculations. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)." us,scenario_114,federal_refundable_credits,2,llm_error,credit_phaseout,False,,"EITC phaseout uses adjusted gross income or earned income, whichever is greater, rather than wages alone. This filer’s partnership/S-corp income, taxable retirement distributions, pension, taxable Social Security, and interest raise AGI far above the childless EITC limit, so every refundable credit component is zero." us,scenario_114,payroll_tax,4,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes employee Social Security and employee Medicare taxes on the $22,700 of wages, while self-employment tax is reported separately and Virginia adds no mandatory employee payroll tax here. The stated components sum to $1,407.40 + $329.15 = $1,736.55; arbitrary adjustments, duplication, and invented state payroll taxes produce the wrong answers." us,scenario_114,self_employment_tax,18,llm_error,payroll_tax_base,False,,"The household's only business-type input is $24,362 of partnership or S-corp income, and that amount is not net earnings from self-employment. S-corp pass-through income is never subject to SE tax. A partnership distributive share counts only for a general partner, and the prompt says any unlisted status is false, so no general-partner status can be assumed. With zero self-employment income, SE tax is $0. Every substantive wrong answer made the same mistake: each model counted the partnership/S-corp amount as SE earnings, then applied 15.3% to 92.35% of it, which gives about $3,442. Some models then added further arithmetic errors on top of that." -us,scenario_114,state_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The Virginia tax is 720 + 5.75% × ($65,593.83 − $17,000) = $3,514.15. Virginia taxable income is the $86,743.77 VA AGI, after the Social Security subtraction, minus two items. The first is $19,419.94 of itemized deductions: charitable gifts of $10,913.78 after the 2026 federal floor of 0.5% of AGI, a medical deduction of $4,441.86, and real estate taxes of $4,064.29. The medical figure is the $10,000 of medical expenses plus the $2,434.80 Medicare Part B premium, less 7.5% of AGI; the $2,000 of over-the-counter health expenses is not counted. The second is $1,730 of exemptions ($930 personal plus $800 for age 65 or older), and the $12,000 age deduction phases out to zero above the $50,000 single threshold. The wrong answers mostly fell into one of these traps: taking the VA standard deduction instead of itemizing, leaving out the Part B premium or counting the over-the-counter expenses as medical, skipping the charitable floor, dropping the $800 age exemption, wrongly allowing the age deduction, or not subtracting Social Security. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py). Reference depends on an unlisted input; output excluded from scoring (Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)." +us,scenario_114,state_income_tax_before_refundable_credits,48,prompt_ambiguity,taxable_income_or_deductions,False,,"Virginia starts from federal AGI of $106,572.56 and subtracts the $19,828.80 of taxable Social Security, giving VA AGI of $86,743.77. The age deduction is fully phased out because AFAGI exceeds the $50,000 single threshold by more than $12,000. The frozen engine calculation itemizes $19,419.94, counting an unlisted Part B premium: charitable gifts of $11,447 less the 2026 0.5%-of-AGI floor, real estate tax of $4,064, and medical costs that include the $2,434.80 Medicare Part B premium but exclude the $2,000 of OTC spending, all above the 7.5% floor. Exemptions of $1,730 ($930 personal plus $800 aged) are then subtracted. The models split three ways: some used the standard deduction, some applied the $12,000 age deduction, and some itemized but dropped the Part B premium or the charitable floor, or counted OTC costs. Several also left out the $800 aged exemption or treated it as a credit. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py). Reference depends on an unlisted input; output excluded from scoring (Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)." us,scenario_114,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_115,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and a reported disability do not by themselves place the head in an Alabama Medicaid eligibility category. PolicyEngine assigns medicaid_category = NONE; the 1.47 FPL MAGI amount therefore cannot establish eligibility through a MAGI category, and immigration eligibility alone is insufficient." us,scenario_115,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_115,snap,5,llm_error,thresholds_rates,False,,"SNAP eligibility uses approximately $23,442 of annual household income here: $18,708 of Social Security plus $4,734 of tax-exempt interest. That income exceeds PolicyEngine’s applicable one-person SNAP eligibility limit, so the calculation stops at ineligibility and returns $0; deductions and minimum-allotment rules do not create a benefit for an ineligible household." -us,scenario_115,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Alabama follows its own income definition. It adds federally tax-exempt interest back into Alabama gross income because Alabama taxes interest on other states' municipal obligations. That puts this filer's Alabama AGI at $4,734, while the $18,708 of Social Security stays exempt. Alabama's single standard deduction is $3,000 at this income level, with no age-65 or disability add-on, and the personal exemption is $1,500, for $4,500 total. That leaves $234 of taxable income, taxed at 2% to give $4.68. Every model either dropped the interest from the Alabama tax base or overstated the deductions so they absorbed the full $4,734." +us,scenario_115,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"Alabama exempts Social Security, but its adjusted gross income includes federally tax-exempt interest. Interest on bonds from other states is taxable in Alabama, and PolicyEngine counts all listed tax-exempt interest as Alabama income. That makes Alabama AGI $4,734. The single filer's $3,000 standard deduction plus $1,500 personal exemption total only $4,500, and Alabama gives no extra deduction for age 65 or older. That leaves $234 of taxable income, taxed at 2% = $4.68. Every model either dropped the interest from Alabama income or overstated the deductions so that they wiped out all of it." us,scenario_115,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_116,snap,1,llm_error,categorical_eligibility,False,,"SNAP’s minimum allotment applies only after a household passes the program’s eligibility tests; it cannot override failure of the gross-income test. At $52,800 of annual household income, this household exceeds the applicable 2026 gross-income limit, so the computation stops with no SNAP benefit." us,scenario_116,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Florida did not expand Medicaid, so a childless, non-pregnant 42-year-old has no MAGI adult or parent category, even with MAGI at 0% FPL (veterans benefits are left out of MAGI). Disability alone does not create eligibility. The disabled pathway requires either receiving SSI or passing Florida's aged/disabled income test. The spouse's $52,800 in veterans benefits counts as unearned income under SSI-related rules, which pushes SSI to $0 and puts the spouse far above the aged/disabled income limit, so medicaid_category is NONE. All three models treated disability status as enough to qualify and never applied the income test that comes with the disabled pathway." @@ -891,19 +891,19 @@ us,scenario_116,spouse_medicare_eligible,4,llm_error,age_disability,False,,"Unde us,scenario_117,child3_early_head_start_eligible,5,llm_error,categorical_eligibility,False,,"Early Head Start eligibility requires both an age-qualified child and satisfaction of the program's low-income eligibility rules; being under age 3 is not independently sufficient. Although Child 3 meets the age condition, household income of approximately $250,733 is far above the applicable poverty-based income limit, so the income test produces no eligibility." us,scenario_117,child3_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Age alone does not establish Head Start eligibility; the child must also qualify through an applicable eligibility pathway, principally the program's low-income criteria in this case. This household's approximately $250,733 income exceeds the relevant poverty-based threshold, so Child 3 is not eligible." us,scenario_117,child3_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC child eligibility requires both categorical eligibility and financial eligibility; being age 2 satisfies only the child-category requirement. The household's approximately $250,733 annual income exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household, so Child 3 is not eligible." -us,scenario_117,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The 2026 return for this married-filing-jointly household uses these rules: AGI of $250,733.36 (wages plus the taxable state refund minus the capital loss), and the full $25,000 OBBBA qualified-overtime deduction, because the joint phaseout starts at $300,000 MAGI. It takes the $32,200 standard deduction plus the new $2,000 non-itemizer charitable deduction, which together ($34,200) beat itemizing. That gives taxable income of $191,533.36 and $31,561.34 of tax under the permanent TCJA brackets. The household then subtracts $6,600 of CTC ($2,200 × 3 children under 17). There is no CDCC because Head has zero earned income. Models went wrong in five main ways: they left out the overtime deduction or capped or phased it out incorrectly, applied TCJA-sunset law, assumed mortgage interest from the loan balance, missed the $2,000 non-itemizer charitable deduction, or claimed a CDCC that the zero-earner spouse blocks. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." +us,scenario_117,federal_income_tax_before_refundable_credits,48,prompt_ambiguity,taxable_income_or_deductions,False,,"Under 2026 OBBBA law, AGI is $250,733. The household takes the $32,200 joint standard deduction plus the $2,000 charitable deduction for non-itemizers, which beats itemizing (charity above the 0.5% AGI floor, plus Arkansas income tax and about $741 of medical). It also takes the full $25,000 qualified-overtime deduction, because the joint phaseout starts at $300,000 MAGI. That gives taxable income of $191,533 and tax of $31,561 at the 10/12/22% brackets. The only nonrefundable credit is the $6,600 CTC (3 × $2,200). There is no CDCC because Head has no earned income. The models went wrong in four main ways: they left out or cut back the overtime deduction, used the old pre-TCJA rules, missed the $2,000 non-itemizer charitable deduction, or claimed a CDCC even though one spouse earns $0. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." us,scenario_117,federal_refundable_credits,5,llm_error,other,False,,"The refundable CTC (ACTC) is capped at the part of the child tax credit left over after the credit has offset income tax. This couple files jointly with about $250,733 of AGI. That is below the $400,000 CTC phase-out, so the full $2,200-per-child CTC ($6,600) is allowed. Their federal income tax is well over $25,000, which absorbs the whole $6,600 as a nonrefundable credit and leaves nothing to refund. EITC is zero at this income, and no other refundable credit applies, so the total is $0. The wrong answers skipped the tax-liability limit and treated the per-child refundable cap ($1,700 or $1,800), or a made-up fraction of it, as money the family receives automatically." us,scenario_117,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is individual, not household-wide: living with a two-year-old does not make the 44-year-old head categorically eligible. The household's approximately $250,733 annual income also exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household." us,scenario_117,payroll_tax,27,llm_error,thresholds_rates,False,,"Only the spouse has wages ($248,675, which already include overtime). Employee Social Security tax is 6.2% up to the 2026 wage base of $184,500, which comes to $11,439. Medicare is 1.45% of all wages, $3,605.79. Combined wages are under the $250,000 MFJ Additional Medicare threshold, and Arkansas has no employee payroll tax, so the total is $15,044.79. Most models used an outdated or guessed Social Security wage base, such as $168,600 (2024) or $176,100 (2025). Others counted the FLSA overtime premium twice, skipped the SS cap, or submitted a number that does not match their own arithmetic." us,scenario_117,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_117,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is determined individually and requires both membership in an eligible category—pregnant, postpartum, breastfeeding, infant, or child under five—and satisfaction of the income or adjunctive-eligibility rules. The spouse is not placed in any eligible WIC category by the stated facts, and household income of approximately $250,733 also exceeds the applicable 185% federal poverty guideline." -us,scenario_117,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"The reference calculation starts from Arkansas AGI of $248,144.53. That figure is wages plus the $530 capital loss. It excludes the $2,589 state refund and keeps the FLSA overtime in income. Arkansas itemized deductions are only the charitable gift after the 2026 floor of 0.5% of AGI: $25,900 minus $1,253.67 equals $24,646.33. There is no mortgage interest because only a loan balance is given, medical costs are below the floor, and employee expenses are below the 2% floor. That leaves taxable income of $223,498.20. The 2026 Arkansas rate schedule produces $8,189.53 of tax on that income. Subtracting $205 of nonrefundable credits ($145 in personal credits plus a $60 additional credit for qualified individuals) gives $7,984.53. The wrong answers split three ways: some skipped the charitable itemization or added imputed mortgage interest, SALT or overtime exclusions; some used old 4.4%/4.7%/4.9% top rates or a high-income schedule that overstates bracket tax; and every model left out the $60 additional credit, with many also omitting or miscounting the $145 personal credits." +us,scenario_117,state_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"Arkansas AGI is $248,144.53: wages minus the $530 capital loss. The $2,589 state refund is not Arkansas income, and the FLSA overtime premium is already counted inside wages and gets no state exclusion. The household itemizes only the charitable gift, reduced by the 2026 floor of 0.5% of federal AGI ($1,253.67), for $24,646.33. No mortgage interest is deductible because only a loan balance is given, medical costs fall under the floor, and employee business expenses fall under the 2% floor. The engine's 2026 schedule taxes $223,498.20 at $8,189.53 (equal to 3.7% of taxable income minus $79.90). Subtracting $205 of nonrefundable credits ($29 × 5 = $145 personal, plus a $60 additional credit) leaves $7,984.53. The wrong answers mostly used the standard deduction or made up mortgage interest, used 3.9%/4.4% schedules that overstate the bracket tax, counted the refund or excluded overtime, or claimed only the $145 personal credits." us,scenario_117,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_118,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The head is 74, blind, and disabled. Their only income is $2,800 a year in Social Security, and they hold $70 in resources. That qualifies them for SSI: countable income after the $20 general exclusion is about $213/month, which is under the federal benefit rate, and the engine pays $9,368 a year. In New York, SSI receipt brings Medicaid eligibility automatically through the SSI-recipient category, and the head also meets NY's aged/disabled income and asset limits. Both models missed that the SSI pathway makes the head categorically eligible. They also failed to check the very low income and assets against the aged/blind/disabled limits." us,scenario_118,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earned compensation, not Social Security retirement benefits. With no wages, self-employment income, or other covered earnings listed, the employee-side payroll-tax base is zero and payroll tax is $0." -us,scenario_118,snap,47,prompt_ambiguity,taxable_income_or_deductions,False,,"The question lists $2,800 of Social Security. That is far below the 2026 federal SSI benefit rate of $994/month, so the head also receives SSI of $780.67/month ($994 − ($233.33 − $20)). For SNAP, SSI is countable unearned income, so gross income is $1,014/month, not $233. SSI receipt also makes the household categorically eligible. After the $209 standard deduction, adjusted income is $805. PolicyEngine's uncapped elderly/disabled excess shelter deduction then brings net income down to $194.38: the shelter costs are about $1,013/month, which is the $598.75 of mortgage interest and property tax plus a utility allowance. That gives $298 − $58.20 = $239.80/month for January–September and $248.58/month for October–December under FY2027 values, or $2,903.94 for the year. Most models either dropped SSI from income, which pushed net income to about zero and the benefit toward the maximum, or counted SSI but used only mortgage interest and property tax as shelter costs, giving net income of $608.75 and about $1,384/year. Some answered $0 because they misread eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)." +us,scenario_118,snap,48,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is blind and disabled and has only $2,800 of Social Security, so they receive $780.67/month of SSI ($994 FBR minus $213.33 countable SS). SNAP counts that SSI as unearned income, making gross income $1,014/month. Subtract the $209 standard deduction to get $805. Then subtract the uncapped elderly/disabled excess shelter deduction of about $611/month, which builds the New York standard utility allowance into shelter costs alongside mortgage interest and property tax. That leaves $194 of net income, a $58.20 contribution, and $298 − $58.20 = $239.80/month for January–September, then $248.58/month for October–December under FY2027 parameters, for $2,903.94 in total. Most models made one of two mistakes. Some dropped SSI, which drives net income to $0 and gives the roughly $3,500 maximum-allotment answer. Others counted SSI but left the utility allowance out of shelter costs, which gives net income of about $609 and an answer near $1,384. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)." us,scenario_118,ssi,36,llm_error,thresholds_rates,False,,"The head is categorically eligible as aged and blind. Countable resources are $70. Countable unearned income is $2,800 minus the $20/month general exclusion ($240/year), which leaves $2,560/year ($213.33/month). The federal SSI payment is the 2026 individual FBR of $994/month minus $213.33, or $780.67/month, which comes to $9,368/year. The `ssi` output is federal SSI only, so no New York State Supplement is added. Models missed this in three main ways: they used a stale or wrong FBR ($943, $967, $983, $990, $991, $993), they added a NY SSP on top of federal SSI, or they applied the $20 exclusion only once per year. Others wrongly found the head ineligible even though $213.33/month of countable income is far below the FBR." -us,scenario_118,state_refundable_credits,35,llm_error,state_local_rule,False,"PolicyEngine's NY real property tax credit measures household gross income without nontaxable Social Security and SSI. Tax Law §606(e)(1) and Form IT-214 (line 10 Social Security; line 12 public assistance/SSI) include both. That puts this 65+ filer in the ≤$1,000 band with the $375 cap. Counting the $2,800 of Social Security alone gives the $2,001–$3,000 band and a $341 cap, and adding SSI gives the $12,001–$13,000 band and a $171 cap.","The whole amount comes from New York's real property tax credit (IT-214, Tax Law §606(e)). It is a refundable circuit-breaker credit for homeowners and renters with household gross income of $18,000 or less, and it is paid even when the filer owes no tax. For a 74-year-old homeowner paying about $1,634 in property taxes against almost no income, the property taxes are far above the income-based threshold, so the 65+ maximum caps the credit at $375. Most models only checked the EITC and child credits, assumed a refundable credit needs tax liability or rent, or dismissed homeowners. The rest used the under-65 cap, added the nonrefundable household credit, made up a formula, or used a lower income band by counting Social Security and SSI. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule. Reference affirmed (N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025)." +us,scenario_118,state_refundable_credits,36,llm_error,state_local_rule,False,,"The only NY refundable credit here is the real property tax credit (IT-214). Homeowners qualify for it through the real estate taxes they pay; it doesn't depend on rent, earned income, children, or any income tax liability. This 74-year-old filer has federal AGI of $0 and property taxes of $1,633.50, above 3.5% of that AGI, so the flat table in force from 2025 gives $375. Most models wrongly tied NY refundable credits to EITC, children, or tax liability and answered $0. The models that did find the credit made errors on the cap, the income band, or the formula, or imputed SSI that wasn't listed, or added the nonrefundable household credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule. Reference affirmed (N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025)." us,scenario_119,child1_chip_eligible,17,llm_error,health_coverage,False,,"Child 1 is 14, so the age test passes. Household MAGI of about $52,569 (roughly 192-197% FPL) is also inside Virginia's FAMIS band, so the income test passes too. The child fails CHIP's categorical test instead. The household facts say the child has employer-sponsored insurance, and a CHIP targeted low-income child cannot be covered under a group health plan. Every answering model ran only the age and income-band screen and ignored the child's ESI coverage flag. None of them applied the no-other-coverage requirement, so all returned eligible." us,scenario_119,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -917,7 +917,7 @@ us,scenario_119,child2_medicaid_eligible,5,llm_error,thresholds_rates,False,,"Th us,scenario_119,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_wic_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,federal_income_tax_before_refundable_credits,21,llm_error,thresholds_rates,False,,"For 2026, OBBBA keeps the TCJA structure. That means a $24,150 head-of-household standard deduction, no personal exemptions, and a $2,200 CTC for each child under 17, so this household's CTC is $4,400. The CDCC rate at this AGI is 35%, giving $546. Tax before credits is $3,056.23. The CDCC and the $200 Saver's credit bring it down to $2,310.23. The nonrefundable CTC is limited only by that remaining liability, so the $4,400 credit wipes it out and the result is $0. The wrong answers came from four mistakes: assuming TCJA expired (personal exemptions and a $1,000-per-child CTC), filing as single, reserving part of the CTC for the refundable ACTC before applying the nonrefundable part, or leaving out credits." -us,scenario_119,federal_refundable_credits,47,reference_engine_defect,credit_phaseout,False,"PolicyEngine's EITC uses the full $55,000 of gross wages as earned income: 7,316 − 0.2106 × (55,000 − 23,890) = 764.23. Its own AGI of $52,569 excludes the $3,087 traditional 401(k) deferral. IRC §32(c)(2)(A)(i), as amended by EGTRRA 2001, counts wages only if they are includible in gross income, and §402(e)(3) deferrals are not. Under that rule the phase-out base is max(AGI $52,569, earned income $51,913) = $52,569, which gives an EITC of $1,276.20 and a total of $3,365.97.","The refundable total is a $2,089.77 ACTC plus a $764.23 EITC. The 2026 OBBBA CTC is $2,200 per child, or $4,400. It first offsets a net liability of $2,310.23: regular tax of $3,056.28 on $28,419 (AGI of $52,569 minus the $24,150 HoH standard deduction), less a $546 CDCC (35% of $1,560) and a $200 Saver's Credit. The remaining $2,089.77 is fully refundable because it is under both the $1,700-per-child cap and the 15%-of-earnings limit. The EITC stays positive because the $7,316 two-child maximum phases out at 21.06% above $23,890 on a $55,000 earnings base. Models missed by zeroing the EITC, assuming tax absorbed the whole CTC, using a $2,000 or $1,000 per-child CTC, applying the old 20% CDCC rate, omitting the Saver's Credit, subtracting employer insurance premiums from AGI, using the wrong filing status, or phasing out the EITC on AGI. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." +us,scenario_119,federal_refundable_credits,48,llm_error,credit_phaseout,False,,"The $3,366.07 total is a $1,276.30 EITC plus a $2,089.77 refundable CTC. The first trap is the EITC: for a 2026 head-of-household filer with two children, the credit phases out at 21.06% above $23,890 and does not reach zero until about $58,630, so AGI of $52,568.54 still leaves $1,276.30. Many models set the EITC to zero. The second trap is the refundable CTC. It equals the 2026 $4,400 CTC ($2,200 per child) minus the liability left after the $546 CDCC (35% rate) and the $200 saver's credit, which is $2,310.23. Models that used $2,000 or $1,000 per child, a 20% CDCC, no saver's credit, or no ACTC at all understated it. Several models also took the $6,389 employer health premium out of wages, which understated AGI and overstated the EITC." us,scenario_119,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,head_medicaid_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -929,10 +929,10 @@ us,scenario_119,reduced_price_school_meals_eligible,7,llm_error,taxable_income_o us,scenario_119,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_119,state_income_tax_before_refundable_credits,47,reference_engine_defect,credit_phaseout,False,"The federal EITC of $764.23 matches exactly $7,316 − 21.06% × ($55,000 − $23,890), which means PolicyEngine phased out the EITC on gross wages of $55,000. IRC §32(c)(2)(A)(i) counts compensation as earned income only when it is includible in gross income, so the $3,087 traditional 401(k) deferral is excluded. Phase-out income should therefore be max(AGI $52,569, earned income $51,913) = $52,569, giving a federal EITC of about $1,276, a VA nonrefundable EITC of about $255, and VA tax of about $1,756.7.","The Virginia result takes three subtractions from AGI of $52,568.54: the 2026 standard deduction of $8,750, three $930 personal exemptions ($2,790), and Virginia's child and dependent care expense subtraction ($1,560). That leaves taxable income of $39,468.54 and tax of $2,011.94, which is then reduced by the 20% nonrefundable Virginia EITC. Most models went wrong on the deductions: they used the older $8,500 (or $4,500) standard deduction, left out the $1,560 child-care subtraction (ending at $2,101.67 or $2,116), or subtracted $6,389 of ESI premiums from AGI. Many also skipped the nonrefundable EITC, or invented Virginia child tax credits or child-care credits that do not exist; Virginia handles child care as a deduction, not a credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." -us,scenario_119,state_refundable_credits,15,llm_error,state_local_rule,False,,"Virginia makes filers choose between a nonrefundable EITC equal to 20% of the federal EITC and a refundable EITC. Both versions cannot be claimed. The household has federal AGI of about $52.6k after the traditional 401(k) and IRA deductions and a federal EITC of about $1.3k. Its Virginia liability is roughly $2,000 (standard deduction and three exemptions, then 5.75% on income above $17k). That liability absorbs the entire nonrefundable credit. PolicyEngine therefore puts the Virginia EITC among the nonrefundable credits that reduce state_income_tax_before_refundable_credits, so va_refundable_credits and state_refundable_credits are $0. Every model that answered treated Virginia's EITC as automatically refundable and put a percentage of the federal EITC into refundable credits, with rates and federal bases that differed from model to model." +us,scenario_119,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"Two Virginia-specific steps produce the reference. First, Virginia taxable income of $39,468.54 comes from federal AGI of $52,568.54 minus the $8,750 2026 standard deduction, three $930 exemptions, and a $1,560 child and dependent care expense deduction. Virginia treats child care as a deduction, not a percentage credit. Second, the $2,011.94 tax is reduced by the nonrefundable Virginia EITC, which is 20% of the $1,276.30 federal EITC, or $255.26. Most models left out the child care deduction, the nonrefundable EITC, or both. Many also used an outdated standard deduction ($8,500, $4,500 or $3,000), subtracted the $6,389 ESI premiums from AGI again, or made up Virginia child care or child tax credits." +us,scenario_119,state_refundable_credits,16,llm_error,state_local_rule,False,,"Virginia makes filers choose one form of its earned income credit: a nonrefundable credit worth 20% of the federal EITC, or the refundable version. The household cannot claim both. This household has about $52.6k federal AGI. After the HoH standard deduction and three exemptions, Virginia tax before credits is roughly $2,000, which is far more than any VA EITC amount. So the full nonrefundable credit gets used and is the better choice. PolicyEngine claims the nonrefundable credit, which reduces state_income_tax_before_refundable_credits, and leaves the refundable EITC and state_refundable_credits at $0. Every model that answered treated the VA EITC as a refundable credit and never compared the two options against the household's Virginia tax liability." us,scenario_119,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_120,federal_income_tax_before_refundable_credits,47,reference_engine_defect,taxable_income_or_deductions,False,,"For 2026, the reference uses the OBBBA law, which made the TCJA structure permanent. Single brackets are 10/12/22/24%, with the 24% bracket running from $105,700 to $201,775, and there is no personal exemption and no 2%-floor miscellaneous deduction. AGI is $232,635. The state refund is not included, and the farm rent loss and the $28 IRA deduction are taken above the line. Itemized deductions are $35,717: SALT of $13,908 (CT income tax plus property tax, under the $40,400 cap), mortgage interest of $2,808 (3,744 × 750k/1M), and charity of $19,001 after the 0.5%-of-AGI floor. That gives taxable income of $196,918 and $39,858 of bracket tax, plus $163.40 of NIIT on $4,300 of interest and rental income. Most wrong answers came from assuming the TCJA expired in 2026, using the old $10,000 SALT cap, leaving CT income tax out of SALT, skipping the debt limit or the charitable floor, subtracting ESI premiums from wages, or dropping or shrinking the NIIT. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." +us,scenario_120,federal_income_tax_before_refundable_credits,48,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 OBBBA law, under which the TCJA rate schedule is permanent. AGI is $232,635 and leaves out the $1,750 state tax refund. Itemized deductions are $35,717: SALT of $13,908 (Connecticut income tax of about $11,896 plus $2,012 property tax, under the $40,400 cap), mortgage interest of $2,808 after the $750k/$1M debt limit, and charity of $19,001 after the new 0.5%-of-AGI floor. That leaves $196,918 of taxable income and $39,858.42 of bracket tax, plus $163.40 of 3.8% NIIT on the full $4,300 of interest and rent. The wrong answers fall into four groups: assuming the TCJA sunset (personal exemption, pre-TCJA brackets, miscellaneous deductions); mis-sizing itemized deductions (the old $10k SALT cap, leaving out CT income tax, no mortgage-debt limit, no charitable floor); subtracting ESI premiums from gross wages; and omitting NIIT or netting the farm loss against it. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_120,federal_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -945,25 +945,25 @@ us,scenario_120,reduced_price_school_meals_eligible,1,parse_contract_failure,mis us,scenario_120,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_120,state_income_tax_before_refundable_credits,47,llm_error,state_local_rule,False,,"Connecticut starts from federal AGI of $234,414. For a single filer above the $75,000 AGI threshold, Connecticut taxes no more than 25% of Social Security benefits ($10,475), so it subtracts $25,140 of the $35,615 federally taxable amount. It also subtracts the $1,750 state tax refund, leaving CT AGI of $207,524. The 2024+ single schedule (2%/4.5%/5.5%/6%/6.5%) gives $11,239.06 on that amount. Connecticut then adds a $250 phase-out add-back, $250 of low-tax recapture and $180 of middle-tax recapture, for a total of $11,919.06. The personal exemption, personal credit and property tax credit are all zero at this income. The wrong answers mostly fall into a few groups: they skipped the 25%-of-benefits Social Security rule (either no subtraction or a full one), used the old 3%/5% bottom brackets, omitted some or all of the $680 add-back and recapture, or claimed exemptions, deductions or credits that Connecticut does not allow at this income." +us,scenario_120,state_income_tax_before_refundable_credits,48,llm_error,state_local_rule,False,,"The trap is Connecticut's Social Security rule for single filers above the $75,000 AGI threshold. CT taxes no more than 25% of total benefits, so the subtraction is federally taxable SS minus 25% of benefits: $35,615 − $10,475 = $25,140. Add the $1,750 SALT refund subtraction and CT AGI comes to $207,523.97. The 2026 single schedule (2%/4.5%/5.5%/6%/6.5%) gives $11,239.06, and three separate add-backs bring it to $11,919.06: the $250 phase-out of the 2% bracket, $250 of low-tax recapture and $180 of middle-tax recapture. Most models either took no SS subtraction, subtracted all taxable SS or all benefits, used the old 3%/5% rates, applied exemptions, deductions or credits that are fully phased out, or left out some or all of the $680 in add-backs and recapture." us,scenario_120,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_121,federal_income_tax_before_refundable_credits,4,prompt_ambiguity,taxable_income_or_deductions,False,,"Two facts drive the $0: PolicyEngine's `disability_benefits` input is a non-taxable income source excluded from AGI, so AGI is wages only ($25,665), and the household's medical costs ($2,200 non-Medicare-Part-B premiums + $30,000 other medical + $400 over-the-counter = $32,600) generate an itemized medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675. That itemized total exceeds AGI, so taxable income floors at $0 and there is no tax to reduce with nonrefundable credits. Every wrong model either taxed the disability benefits, ignored the medical itemized deduction, or both, leaving a positive 10%/12%-bracket residue. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 875.34. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." us,scenario_121,federal_refundable_credits,3,llm_error,credit_phaseout,False,,"The childless EITC phaseout must use the greater of earned income or adjusted gross income. The $13,044 of disability benefits raises AGI above the 2026 childless EITC income limit, so the EITC and total federal refundable credits are $0; all three models instead treated $17,276 of net wages as both earned income and AGI." us,scenario_121,head_medicaid_eligible,11,llm_error,categorical_eligibility,False,,"The trap is that the `is_disabled` flag by itself opens no Medicaid category in South Carolina. The non-MAGI disability route runs through SSI receipt, and SSI here is $0 because $25,665 in wages plus $13,044 of disability benefits leave countable income far above the SSI federal benefit rate (~$12,000/year) after the $20 general and $65-plus-half earned disregards. The MAGI route fails too: MAGI counts only the taxable wages ($25,665 = 1.61 x FPL for a one-person unit, since child support and the disability benefits are excluded), which sits above SC's parent/caretaker limit, above the 138% FPL expansion limit SC has not adopted, and is moot anyway because no child is present to establish a parent/caretaker category. Every wrong model substituted a program the engine does not implement and that this income fails regardless — a working-disabled buy-in at 250% FPL, a medically needy spend-down against the $30,000 of medical expenses, or a bare disability presumption — instead of testing an actual pathway, so medicaid_category is NONE and eligibility is False." -us,scenario_121,head_medicare_eligible,16,llm_error,age_disability,False,,"Under age 65, the Medicare disability pathway requires 24 months of Social Security Disability Insurance entitlement under 42 U.S.C. 426(b), or ESRD or ALS. A disability flag alone does not qualify. The head's $13,044 is generic disability-benefits income, not Social Security disability. SSDI receipt and its duration are not listed, so they count as 0, and ESRD is false. That leaves only the age-65 test, which a 53-year-old fails. All 16 models treated ""is disabled"" plus generic disability benefits as SSDI Medicare entitlement. Several also assumed the 24-month waiting period had already passed, even though the prompt says to treat unlisted inputs as 0 and not to infer benefit receipt." +us,scenario_121,head_medicare_eligible,17,llm_error,age_disability,False,,"PolicyEngine grants Medicare eligibility under 65 only through Social Security Disability Insurance (SSDI) receipt lasting at least 24 months (months_receiving_social_security_disability ≥ 24) or through end-stage renal disease (ESRD). The listed ""disability benefits"" amount is a generic disability-income input, not Social Security disability. No months of SSDI receipt were given, so under the prompt's rules that count is 0, and being flagged ""is disabled"" does not by itself make someone Medicare-eligible. The head is 53, has no qualifying SSDI duration and no ESRD, so they are not eligible. Every model treated the disabled flag plus generic disability income as SSDI and assumed the 24-month waiting period was already met, even though the prompt forbids inferring unlisted benefit receipt." us,scenario_121,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_121,payroll_tax,7,llm_error,payroll_tax_base,False,,"The listed employer-sponsored insurance premium is not specified as a pre-tax payroll deduction, so it does not reduce FICA wages under the prompt’s rule that unlisted statuses are false. Applying 6.2% Social Security and 1.45% Medicare to the full $25,665 of wages yields $1,591.23 and $372.14, totaling $1,963.37." us,scenario_121,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_121,snap,30,llm_error,age_disability,False,,"Every model with an answer let the head's ""is disabled"" flag put the household on SNAP's elderly/disabled track. That track skips the gross income test and allows the uncapped excess medical deduction. But SNAP defines a disabled member by benefit receipt under 7 CFR 271.2: SSI, Social Security disability, or comparable government disability or blindness payments. This head gets no SSI and no Social Security disability, only generic disability benefits, so the household is an ordinary one-person unit. Its gross income is about $3,326 a month ($25,665 wages + $13,044 disability benefits + $1,200 child support, divided by 12), well above South Carolina's gross limit of 130% of the poverty line (about $1,696 a month in FY2026), so SNAP is $0 and the $30,000 in medical costs never matters." +us,scenario_121,snap,31,llm_error,age_disability,False,,"The models treated the head's general ""is disabled"" flag as making this a SNAP elderly/disabled household. Under SNAP's definition (7 U.S.C. 2012(j)), that status depends on receiving qualifying government disability benefits such as SSI or Social Security disability. The head has only generic disability benefits and earns about $2,139/month in wages, which is above the SGA level, so PolicyEngine applies the gross income test. Countable gross income is $39,909/yr ($3,326/mo). That is above both SC's BBCE limit of 200% FPL (about $2,608/mo for one person) and the standard 130% limit, so SNAP is $0. Because the household is not SNAP elderly/disabled, the uncapped excess medical deduction that every model relied on never comes into play." us,scenario_121,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_121,state_income_tax_before_refundable_credits,8,prompt_ambiguity,taxable_income_or_deductions,False,,"South Carolina's tax base starts from federal taxable income, and this filer's deductible medical outlays total $32,600 ($30,000 other medical expenses + $2,200 health insurance premiums excluding Medicare Part B + $400 over-the-counter health expenses). Netted against the 7.5%-of-AGI floor (at most $2,903 on the largest possible AGI of $38,709), that leaves roughly $30,000 of itemized medical deduction — nearly double the $16,100 single standard deduction — which wipes out federal taxable income, and SC's subtraction for the disabled filer's disability income removes anything left, so SC taxable income never reaches the top of the zero-rate bracket and the tax is $0. Every wrong model defaulted to the standard deduction and treated the $30,000 medical line as an irrelevant household fact; several compounded that with invented SC rate schedules and a nonexistent personal exemption. The gap between $0 and every submitted number is entirely the unclaimed itemized medical expense deduction — no bracket or rate correction closes it. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 212.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." -us,scenario_122,federal_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"AGI is $70,646.18. That is $43,446 of non-Social Security income, after the $1,519 net capital loss, plus the 85% cap of $27,200 in taxable Social Security. Under the OBBBA rules for 2026, the deductions are: the $16,100 standard deduction, plus two $2,050 aged/blind amounts, plus the $6,000 senior deduction (MAGI is under $75,000), plus the new $1,000 charitable deduction for single non-itemizers. That totals $27,200, which leaves $43,446.18 of taxable income. The $1,824 of qualified dividends falls in the 0% band, and the $41,622.18 of ordinary income is taxed at 10%/12%, giving $4,746.66. Wrong answers came from four errors: assuming TCJA expired (personal exemption, 15% bracket, miscellaneous itemized deductions), leaving out the senior deduction or the $1,000 non-itemizer charitable deduction, mishandling the Social Security or dividend income inclusion, or zeroing out a clearly positive liability." +us,scenario_122,federal_income_tax_before_refundable_credits,41,llm_error,taxable_income_or_deductions,False,,"Tax year 2026 falls under OBBBA, which made the TCJA structure permanent. That means no personal exemption and no miscellaneous itemized deductions. It also means a $16,100 single standard deduction plus $2,050 each for age and blindness ($20,200), plus the $6,000 senior deduction (MAGI of $70,646 is under $75,000), plus the new $1,000 non-itemizer charitable deduction, for $27,200 in total deductions. With 85% of Social Security taxable ($27,200), AGI is $70,646.18 and taxable income is $43,446.18. The $1,824 of qualified dividends is taxed at 0%, and the $41,622.18 of ordinary income is taxed at 10%/12% (with the 10% bracket ending at $12,400), giving $4,746.66. Most models either applied a TCJA sunset (personal exemption, 15%/25% brackets, itemizing with miscellaneous deductions), left out the senior deduction and/or the $1,000 charitable deduction, or claimed zero tax." us,scenario_122,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_122,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Age, blindness, and disability do not by themselves establish Medicaid eligibility under Minnesota’s aged, blind, or disabled pathways. The head qualified through no Medicaid category, received no SSI, and had MAGI equal to 4.85 times FPL, so the eligibility result is No." us,scenario_122,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary earnings, not Social Security benefits, IRA distributions, pensions, dividends, interest, or capital gains. Because the prompt lists no wages and requires unlisted numeric inputs to be zero, the employee payroll-tax base is zero and payroll tax is $0." us,scenario_122,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_122,state_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"Minnesota starts from federal AGI of $70,646.18. At this income a single filer gets the full Social Security subtraction of $27,200, the standard deduction with both the age and blindness add-ons ($19,300), and the non-itemizer charitable subtraction ($4,977.18). That leaves $19,169 of taxable income, all taxed in the 5.35% bracket, for $1,025.54. Wrong answers fell into three groups. The highest ones limited or skipped the Social Security subtraction and pushed income into the 6.8% bracket. The $0 answers made up senior or pension subtractions, personal exemptions, or credits that zero out the tax. Most of the near misses left out the charitable subtraction or misjudged the age/blind standard deduction." +us,scenario_122,state_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"Minnesota starts from federal AGI of $70,646.18. Because AGI is below the single phaseout threshold, the full $27,200 of federally taxable Social Security is subtracted. The filer then takes the $19,300 aged-plus-blind standard deduction and the non-itemizer charitable subtraction of $4,977.18, which leaves $19,169 taxed entirely at 5.35% ($1,025.54), with no nonrefundable credits. The wrong answers fall into four groups: models that skipped or only partly applied the Social Security subtraction and pushed income into the 6.8% bracket (~$2,400–3,200); models that dropped the charitable subtraction (~$1,250–1,400); models that invented senior or pension subtractions or credits and landed at $0; and near-misses that overstated the charitable subtraction at $5,154 or added back tax-exempt interest." us,scenario_122,state_refundable_credits,7,llm_error,credit_phaseout,False,,"The only candidate Minnesota refundable credit is the renter's credit. There are no earnings or children, so the Working Family Credit and Child Tax Credit do not apply. The household's income is about $77,460: AGI plus nontaxable Social Security and tax-exempt interest. At that income the 2026 renter's credit schedule gives nothing, so mn_refundable_credits and state_refundable_credits are both $0. Every model that answered assumed the household falls inside a paying bracket of the renter's credit schedule and granted $1,000-$2,300 from 17% of the $28,235 rent. None of them noticed that the credit is fully phased out at this income." us,scenario_123,child1_chip_eligible,4,llm_error,thresholds_rates,False,,"Child 1 is 16, so the CHIP age rule (under 19) is met. What decides the case is the income test. The child has to file a return on $45,000 of wages, so that income counts in household MAGI. With the parents' income, household MAGI is about $145,002, which is over 500% FPL for a family of three. That is far above Pennsylvania's CHIP income ceiling (314% FPL), and PolicyEngine's is_chip_eligible applies that ceiling and does not treat the unsubsidized full-cost CHIP buy-in as eligibility. Every model that answered 'Yes' skipped the income ceiling: either it assumed PA's full-cost CHIP makes any income eligible, or it looked only at the child's age." us,scenario_123,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -971,7 +971,7 @@ us,scenario_123,child1_head_start_eligible,1,parse_contract_failure,missing_outp us,scenario_123,child1_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_123,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,household_unit_or_filing_status,False,,"The output is the joint return for Head and Spouse. Their AGI is $100,002 (Head's $100,000 wages plus $2 interest). Child 1's $45,000 of wages belong on the child's own return: they are not part of the parents' AGI, and the child's tax is not added to this output. Under 2026 OBBBA law, the parents get the $32,200 MFJ standard deduction plus the new $2,000 non-itemizer charitable deduction, leaving $65,802 of taxable income. The 10%/12% brackets (10% bracket ends at $24,800) give $7,400.24 of tax, and the $2,200 CTC is fully nonrefundable against it, for $5,200.24. No qualified overtime deduction applies because Head is not flagged as paid hourly. The models went wrong in four main ways: putting the child's wages in the joint AGI or adding the child's separate tax; assuming the TCJA expired; leaving out the $2,000 non-itemizer charitable deduction; and inventing deductions such as mortgage interest or overtime. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." +us,scenario_123,federal_income_tax_before_refundable_credits,47,prompt_ambiguity,household_unit_or_filing_status,False,,"The output is the joint tax unit's figure. That unit's AGI is $100,002 (head's wages plus $2 interest) because the 16-year-old dependent's $45,000 wages are not part of the parents' AGI, and no separate child return is added. The 2026 OBBBA rules give a $32,200 standard deduction, a $2,000 cash-charity deduction for non-itemizers, a 10% bracket up to $24,800 and a $2,200 CTC. That makes taxable income $65,802, tax $7,400.24, and $5,200.24 after the CTC. The head is not flagged as paid hourly, so there is no overtime deduction, and the TCJA sunset (personal exemptions, 15% bracket, $1,000 CTC) does not apply. Most models either put the child's wages into the joint AGI, added a separate child tax, used pre-TCJA sunset law, or left out the $2,000 non-itemizer charitable deduction. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." us,scenario_123,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The Child Tax Credit is a nonrefundable credit first; the additional child tax credit is only the residual the taxpayer cannot use against liability, further capped at $1,700 per child in 2026 and at 15% of earnings above $2,500. This household's $145,002 of income less the 2026 MFJ standard deduction leaves roughly $112,800 of taxable income and about $14,000 of federal income tax before refundable credits, which absorbs the entire CTC for the 16-year-old, and the one-child EITC is long gone at joint income above $57,000 — so total federal refundable credits are $0. Every wrong model stopped at the earned-income phase-in and the $400,000 MAGI phase-out and paid out a per-child refundable cap ($1,500, $1,600, $1,700, $2,000, or $2,400) as if the ACTC were a standalone entitlement. One model additionally fabricated a $205 EITC at an income level far past its phase-out." us,scenario_123,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -985,6 +985,6 @@ us,scenario_123,spouse_chip_eligible,1,parse_contract_failure,missing_output,Fal us,scenario_123,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid eligibility requires qualification through a specific categorical pathway and its applicable income test. The 39-year-old spouse has no qualifying category and MAGI of 5.31 times FPL, so the spouse fails every available pathway and is not eligible." us,scenario_123,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_123,state_income_tax_before_refundable_credits,47,prompt_ambiguity,household_unit_or_filing_status,False,,"The prompt puts everyone in one tax unit: the head and spouse file jointly, and the 16-year-old is their dependent. The child's $45,000 of wages is not reported on the parents' return, and no separate child return is computed. Pennsylvania's flat 3.07% tax therefore applies only to the couple's $100,000 of compensation plus $2 of interest, with no standard deduction, exemptions, or itemized deductions: $100,002 × 0.0307 = $3,070.06. Almost every model added the child's $45,000, either by pooling it into the household base or by adding a separate child return, and arrived at about $4,451.56. A few models invented PA deductions or made arithmetic errors. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." +us,scenario_123,state_income_tax_before_refundable_credits,48,prompt_ambiguity,household_unit_or_filing_status,False,,"The prompt puts everyone in one tax unit, and the 16-year-old is a dependent in it. The tax unit's PA tax base is the head and spouse's income only: $100,000 of wages plus $2 of interest, or $100,002. At 3.07% with no deductions and no forgiveness, that gives $3,070.06. The dependent child's $45,000 of wages is not on the parents' return. Almost every model added it anyway, either by putting it into one $145,002 base or by adding a separate $1,381.50 child return, and got about $4,451.56. A smaller group made up PA deductions (standard, charitable, itemized, or federal taxable income), used the wrong rate, or submitted a number that didn't match its own math. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)." us,scenario_123,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv index 8a954674..610e244d 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv @@ -38,7 +38,7 @@ us,scenario_040,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin us,scenario_060,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Texas household in 2026. The household, consisting of a single adult with approximately $62,775 in annual income, exceeds the income eligibility threshold for SNAP benefits. Although the model determined that the household would take up SNAP if eligible (takes_up_snap_if_eligible = True), the high household income disqualifies them from receiving any benefits. The repeated entries in the computation trace reflect PolicyEngine's systematic evaluation across different time periods or scenarios, all consistently yielding zero benefits due to income ineligibility.", us,scenario_004,free_school_meals_eligible,0.0,16,"PolicyEngine determined that this New York household is not eligible for free school meals, resulting in a value of 0. Although the household's income of $52,675 (comprising $51,475 in social security and $1,200 in interest income) places them at 2.43 times the federal poverty guideline, they do not qualify under the categorical eligibility pathway via SNAP or TANF. While New York does have a universal free school meals program, this household does not meet the eligibility criteria for that program. Consequently, the school_meal_tier variable resolved to 0, and with no subsidy amount applied (school_meal_net_subsidy = 0), the free_school_meals variable is 0.", us,scenario_080,federal_refundable_credits,0.0,6,"For this single adult in Pennsylvania with approximately $940 in household income in 2026, PolicyEngine calculated federal refundable credits of $0. The calculation examined five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. With such low income and no children in the household, this individual did not qualify for any of the major refundable federal tax credits available in 2026.", -us,scenario_100,federal_refundable_credits,2878.0966796875,6,"For this Montana head-of-household filer with two children and approximately $5,915 in household income, PolicyEngine calculated total federal refundable credits of $2,878.10. The largest component was the Earned Income Tax Credit (EITC), which contributed $2,365.89 based on the household's low earned income and eligibility as a head of household with two qualifying children. The remaining $512.21 came from the refundable portion of the Child Tax Credit (refundable_ctc), which provides additional support for families with dependent children whose tax liability is below the credit amount. No recovery rebate credits, refundable American Opportunity credits, or refundable payroll tax credits applied to this household in 2026.", +us,scenario_100,federal_refundable_credits,822.2882690429688,71,"PolicyEngine calculated federal refundable credits of $822.29 for this household in 2026, all of it the earned income tax credit. The household has two qualifying children, aged 6 and 5. The filer's $5,914.72 of wages include $3,859.00 of traditional 401(k) deferrals, which are excluded from gross income and so are not earned income for the credit; earned income is $2,055.72. The credit phases in at 40% of earned income, $822.29, which is below the $7,316.00 maximum for two children. The refundable child tax credit is $0.00: it is 15% of earned income above $2,500, and earned income is below that.", us,scenario_060,ssi,0.0,129,"The 70-year-old head of household in Texas receives $0 in SSI benefits in 2026 despite meeting the basic eligibility criteria. The individual qualifies as aged (age 70), passes the resource test with $180 in countable resources, and would receive a maximum SSI payment of $994 if eligible. However, the countable income of $5,053.75 per month—derived from $3,000 in earned income and $59,775 in unearned income—far exceeds the SSI income limit, resulting in a negative uncapped SSI amount of -$4,059.75. This excess income eliminates the benefit entirely, yielding a final SSI payment of $0.", us,scenario_025,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Ohio household in 2026. TANF is a needs-based program designed to provide cash assistance to low-income families with children, and this household's eligibility was determined based on its composition and income level. Since the household consists of two adults with no children, it does not meet the basic eligibility requirements for TANF, which requires the presence of a dependent child in the household. Additionally, with a household income of approximately $94,925, the household's earnings substantially exceed the income thresholds for TANF assistance in Ohio. Consequently, PolicyEngine correctly determined that no TANF benefit applies to this household.", us,scenario_118,state_refundable_credits,375.0,50,"PolicyEngine calculated state refundable credits of $375 for this New York resident by evaluating refundable credit programs across all 50 states and the District of Columbia. Since the household is located in New York (NY = True), only the ny_refundable_credits component was non-zero at $375, while all other state programs returned $0. This $375 credit derives entirely from New York's real property tax credit, which the household qualifies for based on three factors: the household member's age of 74, real estate taxes paid of $1,633.50, and New York residency. The real property tax credit is designed to provide relief to eligible taxpayers on their property tax burden, and PolicyEngine determined the household's benefit under this program to be $375 for the 2026 tax year.", @@ -64,13 +64,13 @@ us,scenario_118,ssi,9368.0,128,"PolicyEngine calculated an annual SSI benefit of us,scenario_005,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this California joint household in 2026 because neither spouse reported self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this household's approximately $585,777 in income derives from other sources (such as wages, investments, or other non-self-employment sources), there is no self-employment income base upon which to calculate the 15.3% self-employment tax rate. Consequently, both individual entities in the household show $0 self-employment tax liability, resulting in a combined household value of $0.", us,scenario_042,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Wisconsin household because the household has no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $58,434 in total income derives from sources other than self-employment (such as wages or salary), there is no self-employment income base upon which to apply the 15.3% self-employment tax rate. Consequently, PolicyEngine correctly determined that no self-employment tax liability exists for this household in tax year 2026.", us,scenario_080,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Pennsylvania resident with ~$940 household income in tax year 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with pa_refundable_tax_credits specifically evaluated for this household since the PA state indicator is true. All state-level refundable credit variables, including Pennsylvania's, returned $0, resulting in a total state_refundable_credits value of $0. This outcome reflects that the household does not qualify for any state-administered refundable tax credits based on their income level and household composition.", -us,scenario_100,state_refundable_credits,473.1777038574219,52,"PolicyEngine calculated state refundable credits of $473.18 for this Montana head-of-household with two children and ~$5,915 annual income. Since the household resides in Montana (MT = True), only the Montana state refundable credits component was evaluated, while all other state programs returned zero. The Montana refundable credits derive entirely from the Montana Earned Income Tax Credit (EITC), which computed to $473.18 for the tax unit head. This amount represents 20% of the federal EITC value of $2,365.89, reflecting Montana's standard state EITC rate that provides a percentage of the federal credit to eligible low-income working families.", +us,scenario_100,state_refundable_credits,164.45765686035156,84,"PolicyEngine calculated Montana refundable credits of $164.46 for this household in 2026, all of it the Montana earned income tax credit. Montana's credit is 20% of the federal credit. The federal credit is $822.29: 40% of $2,055.72 of earned income for a filer with two qualifying children, where earned income is the filer's $5,914.72 of wages less $3,859.00 of traditional 401(k) deferrals, which are excluded from gross income. 20% of $822.29 is $164.46.", us,scenario_005,state_income_tax_before_refundable_credits,40975.25390625,91,"PolicyEngine calculated California income tax before refundable credits of $40,975.25 for this joint filer household. Federal AGI is $543,076.00, and California AGI is $538,928.31 after a $4,147.68 SALT-refund subtraction. California itemized deductions include $17,280.00 of real estate taxes, $2,573.53 of charitable deductions, and $2,986.30 of miscellaneous deductions, totaling $22,839.83 before limitation. After a $2,319.90 overall reduction, itemized deductions are $20,519.93, exceeding the $11,412.00 standard deduction. Subtracting the selected $20,519.93 deduction from California AGI gives taxable income of $518,408.38. California income tax before credits is $41,089.25. Subtracting the $114.00 non-refundable exemption credit gives the reference value of $40,975.25.", us,scenario_025,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in Ohio for tax year 2026. The primary reason is that the head is 61 years old, which exceeds CHIP's age eligibility threshold that typically covers children and young adults up to age 18 or 19 depending on state rules. Additionally, even if age were not a barrier, the household income of approximately $94,925 would likely exceed Ohio's CHIP income limits for any eligible category. Since the person fails both CHIP's age requirement and income criteria, PolicyEngine correctly returned is_chip_eligible as False.", us,scenario_118,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this single-adult household in New York in 2026. The household's annual income of approximately $2,800 places it above the income eligibility thresholds for TANF in New York State, which has strict income limits for assistance programs. Additionally, TANF eligibility in New York requires the presence of dependent children in the household, and this benchmark household contains zero children. Since the household fails to meet both the income requirement and the family composition requirement, PolicyEngine correctly determined that no TANF benefit applies.", us,scenario_060,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. The household head is 70 years old, which exceeds CHIP's age eligibility requirements—the program serves children and pregnant women, not seniors. Additionally, with a household income of approximately $62,775, the person does not meet CHIP's income thresholds for the eligible populations. Since the person fails both CHIP's age criterion and income criterion, they are classified outside any eligible CHIP category, resulting in ineligibility.", us,scenario_080,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Pennsylvania household in 2026. The household's annual income of approximately $940 falls below the threshold at which Pennsylvania's local income tax obligations are triggered. Since the household does not meet the income requirements to owe local income tax in their jurisdiction, no tax liability was assessed. This result reflects Pennsylvania's local tax structure, which exempts very low-income households from local income tax obligations.", -us,scenario_042,state_income_tax_before_refundable_credits,284.74090576171875,77,"For this single 79-year-old Wisconsin resident with approximately $58,434 in household income, PolicyEngine calculated a state income tax before refundable credits of $284.74. The calculation began with an adjusted gross income of $43,637.65, from which Wisconsin subtracts $13,655.95 in income subtractions (primarily taxable Social Security income), yielding a Wisconsin AGI of $29,981.71. After applying the Wisconsin standard deduction of $12,671.36 and exemptions totaling $950 (comprising a base exemption of $700 plus an additional $250 age-based exemption for being over 65), the taxable income was reduced to $16,360.34. This taxable income generated a tax before credits of $584.74, which was then reduced by $300 in non-refundable credits (the Wisconsin property tax credit based on real estate taxes of $5,010.88), resulting in the final state income tax before refundable credits of $284.74.", +us,scenario_042,state_income_tax_before_refundable_credits,0.0,101,"PolicyEngine calculated Wisconsin income tax before refundable credits of $0.00 for this 79-year-old filer in 2026. IRS gross income of $51,708.00 comprises $19,200.00 of taxable retirement distributions, $16,846.00 of taxable Social Security, $6,848.00 of dividends, $4,886.00 of taxable private pension income, $3,753.00 of capital gain distributions and $175.00 of taxable interest; a $1,127.29 loss deduction for the farm loss gives federal AGI of $50,580.71. Wisconsin subtracts the $16,846.00 of taxable Social Security and $1,125.90, 30% of the capital gain distributions, giving Wisconsin AGI of $32,608.81. After the $12,461.34 standard deduction and $950.00 of exemptions ($700.00 plus $250.00 for a filer 65 or older), Wisconsin taxable income is $19,197.46 (the cent-rounded figures here sum to $19,197.47) and tax before credits is $708.70, or $408.70 after the $300.00 property tax credit. The filer is 67 or older, and the $24,086.00 of pension income and retirement distributions exceeds the $24,000.00 cap, so Wisconsin's retirement income exclusion (Schedule SB line 16) subtracts $24,000.00. That leaves no Wisconsin taxable income and a tax of $0.00 on that path. The exclusion path's $0.00 is less than the standard path's $408.70, so the filer elects it and the tax before refundable credits is $0.00.", us,scenario_025,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for CHIP (Children's Health Insurance Program) in 2026, resulting in a value of False. The spouse's age of 57 years exceeds CHIP's eligibility criteria, which are designed for children and young adults up to a specified maximum age threshold. Additionally, the household's income of approximately $94,925 places it above CHIP's income limits for the applicable family size and state (Ohio). Since the spouse fails both CHIP's age requirement and income threshold, PolicyEngine correctly classified them as ineligible for the program.", us,scenario_100,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Montana head of household with two children and approximately $5,915 in household income for tax year 2026. Montana does not impose a local income tax at the municipal or county level, which is why the local_income_tax variable resolved directly to zero. This result reflects the state's tax structure, where income taxation occurs only at the state level rather than through local jurisdictions. Consequently, regardless of the household's income level or filing status, no local income tax liability was generated.", us,scenario_118,head_wic_eligible,0.0,28,"PolicyEngine determined that this 74-year-old household member is **not eligible for WIC** despite meeting multiple eligibility criteria. The household's countable income of $12,168 (comprising $2,800 in Social Security and $9,368 in SSI) falls well below the WIC income limit of $29,526, satisfying the income test. The individual also meets categorical eligibility requirements through enrollment in Medicaid and receipt of SNAP benefits ($248.58), and demonstrates nutritional risk status. However, WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children up to age 5—categories defined by WIC program rules. Since this household member is 74 years old, they fall into WIC category 5 (other), which is ineligible regardless of meeting the income, categorical, and nutritional risk criteria.", @@ -117,7 +117,7 @@ us,scenario_005,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spou us,scenario_026,federal_refundable_credits,2013.4892578125,6,"For this North Carolina joint household with three children and approximately $85,209 in annual income, PolicyEngine calculated federal refundable credits of $2,013.49 for tax year 2026. The entire amount derives from the refundable Child Tax Credit (refundable_ctc), which contributed $2,013.49, while other refundable credit components—the Earned Income Tax Credit (eitc), refundable American Opportunity Credit, recovery rebate credit, and refundable payroll tax credit—all yielded zero. The refundable portion of the Child Tax Credit is available to families with qualifying children when their tax liability is insufficient to claim the full credit, allowing them to receive the excess as a refund up to the refundable limit, which explains why this household with three dependent children receives this substantial credit despite their moderate income level.", us,scenario_080,head_medicare_eligible,0.0,1,"PolicyEngine determined that the household head is not Medicare eligible (is_medicare_eligible = False) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65, along with certain disability or end-stage renal disease criteria. Since the benchmark household is a single adult with a household income of approximately $940, the model evaluated the household against Medicare's eligibility requirements and found that the head does not meet the qualifying conditions. The resulting value of False (represented as 0.0 in the reference output) indicates that this household member would not be entitled to Medicare benefits under 2026 rules.", us,scenario_062,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Florida household because the household has no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $37,642 in household income derives from other sources (such as wages or salaries), there is no self-employment income base upon which to apply the 15.3% self-employment tax rate. Consequently, PolicyEngine's computation trace shows self_employment_tax = 0, reflecting the absence of any self-employment activity subject to Social Security and Medicare taxes.", -us,scenario_119,federal_refundable_credits,2854.0087890625,6,"For this Virginia head-of-household with two children and approximately $55,800 in household income, PolicyEngine calculated total federal refundable credits of $2,854.01. This amount comprises two main components: the Earned Income Tax Credit (EITC) of $764.23 and the refundable portion of the Child Tax Credit (refundable_ctc) of $2,089.77. The EITC provides a credit based on the household's earned income and number of qualifying children, while the refundable Child Tax Credit delivers the portion of the $2,000-per-child credit that exceeds any tax liability. No additional refundable credits applied to this household, including the refundable American Opportunity Credit, recovery rebate credit, or refundable payroll tax credit.", +us,scenario_119,federal_refundable_credits,3366.07373046875,169,"PolicyEngine calculated federal refundable credits of $3,366.07 for this head of household filer with two children, aged 14 and 11, in 2026: a $1,276.30 earned income tax credit and a $2,089.77 refundable child tax credit. Earned income for the credits is $51,912.80: $55,000.00 of wages less $3,087.20 of traditional 401(k) deferrals, which are excluded from gross income. With $800.00 of taxable interest and a $144.26 traditional IRA deduction, AGI is $52,568.54. The earned income tax credit phases in to its $7,316.00 maximum for two children and is reduced by $6,039.70, 21.06% of the $28,678.54 by which AGI exceeds the $23,890.00 phase-out start, leaving $1,276.30. The child tax credit is $4,400.00, $2,200.00 per child. Income tax before credits is $3,056.23 on taxable income of $28,418.54 (AGI less the $24,150.00 standard deduction); after the $546.00 child and dependent care credit (35% of $1,560.00 of care expenses) and the $200.00 saver's credit, $2,310.23 of liability remains for the child tax credit to offset. The remaining $2,089.77 is refundable, as it is below both the $3,400.00 refundable maximum and $7,411.92, 15% of earned income above $2,500.", us,scenario_042,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. CHIP eligibility in Wisconsin requires applicants to meet both age and income thresholds, and this 79-year-old individual fails CHIP's age criterion, as the program is designed for children and young adults rather than seniors. Additionally, with a household income of approximately $58,434, the person does not meet CHIP's income requirements for the limited categories of adults that the program covers in some states. The combination of exceeding the maximum age limit and failing to qualify under CHIP's income-based criteria resulted in the ineligibility determination.", us,scenario_062,state_income_tax_before_refundable_credits,0.0,46,"PolicyEngine calculated state_income_tax_before_refundable_credits as $0.00 for this Florida resident with approximately $37,642 in household income. The computation aggregates income tax liabilities across all 50 states plus Washington D.C. and New York City, with each individual state and local jurisdiction contributing $0.00 to the total. Florida has no state income tax, which is the primary reason for the zero value for this household's state of residence. All other states' income tax variables (al_income_tax_before_refundable_credits, ca_income_tax_before_refundable_credits, ny_income_tax_before_refundable_credits, and so forth) also returned $0.00, either because the household does not reside in those jurisdictions or because the income level falls below taxable thresholds in those states.", us,scenario_026,payroll_tax,6518.474609375,5,"PolicyEngine calculated the household's payroll tax liability at $6,518.47 for tax year 2026 by summing the employee payroll tax contributions across all earners in the SPM unit. The employee payroll tax total of $6,518.47 comprises two components: employee Social Security tax of $5,282.95 and employee Medicare tax of $1,235.53. These amounts were derived from the earnings of the tax unit head (the second entity in the household array), who earned sufficient income to trigger both the 6.2% Social Security tax on wages up to the annual cap and the 1.45% Medicare tax on all wages. The remaining household members—the spouse and three children—generated no payroll tax liability, as indicated by the per-entity breakdown showing zero contributions for entities 0, 2, 3, and 4.", @@ -136,7 +136,7 @@ us,scenario_081,federal_income_tax_before_refundable_credits,26932.2734375,58,"P us,scenario_100,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False). The head of household is age 46, which places them outside the age range for CHIP eligibility, as CHIP is a program designed for children. Additionally, the head of household qualifies for Medicaid as a parent (is_medicaid_eligible = True), and since CHIP is intended to cover children who do not qualify for Medicaid, this further establishes ineligibility for the CHIP program. The result of False reflects that this adult household member does not meet the eligibility criteria for the Children's Health Insurance Program.", us,scenario_042,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Wisconsin household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $58,434 places it at 4.10 times the federal poverty guideline, which exceeds the 185% income threshold required for reduced-price meal eligibility. Since Wisconsin does not have universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is assigned to the PAID school_meal_tier and receives no subsidy (school_meal_net_subsidy = 0). Consequently, the household must pay the full price for school meals.", us,scenario_062,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this single-adult household in Florida with an annual income of approximately $37,642 in tax year 2026. The computation trace shows that while the household's `takes_up_snap_if_eligible` parameter was set to True, indicating willingness to participate in the program, the household's income level exceeded the eligibility threshold for SNAP benefits. With an income of roughly $37,642 annually, this single-person household surpassed the income limit that would qualify them for assistance under SNAP's federal guidelines, resulting in zero benefits despite the household's stated participation preference.", -us,scenario_119,state_income_tax_before_refundable_credits,1859.0943603515625,103,"For this Virginia head-of-household filer with approximately $55,800 in household income, PolicyEngine calculated a state income tax liability of $1,859.09 before refundable credits. The calculation began with an adjusted gross income (AGI) of $52,568.54, from which Virginia deductions of $8,750 (the standard deduction for head-of-household filers) were subtracted, along with $1,560 in child dependent care expense deductions and $2,790 in personal exemptions, yielding a taxable income of $39,468.54. This taxable income was then subject to Virginia's tax brackets, resulting in a tax before credits of $2,011.94. Finally, PolicyEngine applied a non-refundable Virginia EITC credit of $152.85 (derived from a potential EITC of $764.23), reducing the final state income tax before refundable credits to $1,859.09.", +us,scenario_119,state_income_tax_before_refundable_credits,1756.681396484375,156,"PolicyEngine calculated Virginia income tax before refundable credits of $1,756.68 for this head of household filer with two children in 2026. Virginia AGI is federal AGI, $52,568.54: $55,000.00 of wages less $3,087.20 of traditional 401(k) deferrals, plus $800.00 of taxable interest, less a $144.26 traditional IRA deduction. Subtracting the $8,750.00 standard deduction (which exceeds the $2,057.36 itemized), $2,790.00 of personal exemptions ($930.00 for each of three people) and a $1,560.00 child and dependent care expense deduction leaves Virginia taxable income of $39,468.54. The tax on that income is $2,011.94. Virginia's non-refundable earned income tax credit is 20% of the federal credit of $1,276.30, $255.26, which leaves $1,756.68. The federal credit counts $51,912.80 of earned income, the wages less the 401(k) deferrals.", us,scenario_026,state_refundable_credits,0.0,45,"PolicyEngine calculated state_refundable_credits as $0 for this North Carolina household in 2026 by summing refundable tax credits across all 50 states and the District of Columbia. Since the household resides in North Carolina, only nc_refundable_credits would be applicable, and this value evaluated to $0. All other state-level refundable credit variables (al_refundable_credits, ar_refundable_credits, az_refundable_credits, and so forth through wv_refundable_credits) also returned $0, as the household does not reside in those jurisdictions. The final aggregation of these 51 jurisdiction-level variables produced the total state_refundable_credits value of $0, indicating that either North Carolina offers no refundable state tax credits for this household's circumstances, or the household does not qualify for any available refundable credits based on income, filing status, or dependent count.", us,scenario_005,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for CHIP (Children's Health Insurance Program) in 2026, resulting in a value of False. CHIP eligibility requires that an individual be under age 19 (or in certain states, up to age 21), but the spouse is 33 years old and therefore exceeds the maximum age threshold. Additionally, the household income of approximately $585,777 substantially exceeds CHIP's income limits, which are typically set at or below 200–250% of the federal poverty level depending on the state. Since the spouse fails both CHIP's age requirement and income eligibility criteria, PolicyEngine correctly classified them as ineligible for the program.", us,scenario_081,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this household with approximately $174,088 in income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. This outcome reflects that the household's income level and composition—a single adult with no children and no qualifying education expenses—do not satisfy the eligibility requirements for any of the available refundable federal tax credits in 2026.", @@ -194,7 +194,7 @@ us,scenario_026,child2_wic_eligible,0.0,1,"PolicyEngine determined that child 2 us,scenario_081,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Massachusetts household in 2026. The household, consisting of a single adult with no children and an annual income of approximately $174,088, does not qualify for TANF benefits because the program is designed to provide cash assistance to families with dependent children. Since this household has zero children, it falls outside the eligibility criteria for TANF, resulting in no benefit payment regardless of income level.", us,scenario_062,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $37,642 translates to a school_meal_fpg_ratio of 3.86, meaning the household income is 386% of the federal poverty guideline—well above the 185% threshold required for reduced-price meal eligibility. Since Florida does not have universal free school meals and the household does not meet categorical eligibility through programs like SNAP or TANF, the household is assigned a school_meal_tier of PAID and receives no subsidy (school_meal_net_subsidy = 0). Consequently, the household must pay the full price for school meals.", us,scenario_100,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this Montana household are eligible for Early Head Start, resulting in a value of [False, False, False]. Early Head Start eligibility in PolicyEngine is typically determined by factors such as the child's age (generally limited to infants and toddlers under age 3) and household income relative to federal poverty thresholds. With a household income of approximately $5,915 and a head-of-household filing status with one adult and two children, the household's circumstances did not satisfy the program's eligibility criteria as modeled for the 2026 tax year. The resulting array indicates that neither child qualified for the program.", -us,scenario_007,state_income_tax_before_refundable_credits,755.7783203125,60,"For this single Idaho resident with approximately $54,565 in household income, PolicyEngine calculated $755.78 in state income tax before refundable credits. The calculation began with an adjusted gross income of $49,064.12, from which $13,784.12 in taxable Social Security benefits were subtracted as an Idaho-specific deduction, resulting in an Idaho AGI of $35,280. After applying the standard deduction of $16,100, the household's Idaho taxable income was reduced to $19,180. Idaho's tax brackets and rates were then applied to this taxable income amount to arrive at the final tax liability of $755.78, with all other states contributing zero tax liability.", +us,scenario_007,state_income_tax_before_refundable_credits,661.3170166015625,76,"PolicyEngine calculated Idaho income tax before refundable credits of $661.32 for this household in 2026: $651.32 of income tax plus the $10.00 permanent building fund tax. Federal AGI of $49,064.12 comprises $35,280.00 of taxable private pension income and $13,784.12 of taxable Social Security. Idaho subtracts $15,864.12: the $13,784.12 of taxable Social Security and a $2,080.00 subtraction for the health insurance premiums the household pays. That gives Idaho AGI of $33,200.00, and subtracting the $16,100.00 standard deduction leaves Idaho taxable income of $17,100.00. The tax on that income is $651.32, and no non-refundable credit reduces it.", us,scenario_043,ssi,0.0,25,"For the 66-year-old head of household in Colorado in 2026, PolicyEngine calculated an SSI benefit of $0.00. Although the individual meets the categorical requirement for SSI as an aged, blind, or disabled person (is_ssi_aged_blind_disabled=True), the household income of approximately $4,642 exceeds the SSI income limit, resulting in ineligibility. PolicyEngine determined that the household would take up SSI if eligible (takes_up_ssi_if_eligible=True), but since the income threshold was exceeded, no benefit amount was payable. SSI is evaluated on an individual basis, and in this case the person's countable income eliminated any SSI entitlement despite meeting the non-financial eligibility criteria.", us,scenario_043,snap,3576.0,460,"PolicyEngine computes 2026 SNAP for this one-person Colorado household as $3,576 for the year, the sum of twelve monthly benefits of $298 each. The household is eligible in every month. Its gross income is $387.35 a month, of which $340.68 is earned, against a SNAP poverty guideline of $1,304.17 from January through September (a ratio of about 30%) and $1,330 from October through December (about 29%). It passes the gross and net income tests and, with its 66-year-old member counted as elderly or disabled, the asset test on $2,800 of bank assets; it is also categorically eligible through Colorado's TANF non-cash eligibility test. Each month its normal allotment equals the one-person maximum allotment for the contiguous states, $298.", us,scenario_026,child3_wic_eligible,0.0,1,"PolicyEngine determined that the third child in this North Carolina household is not eligible for WIC benefits, resulting in a value of False. WIC eligibility is primarily determined by income thresholds, which are set at 185% of the federal poverty line. For a household of five persons in 2026, this income limit would be approximately $47,000 annually. Since this household's income is approximately $85,209, it exceeds the WIC income threshold by a substantial margin, placing the household above the maximum allowable income for program participation. Consequently, PolicyEngine marked all household members, including the third child, as ineligible for WIC benefits.", @@ -1111,7 +1111,7 @@ us,scenario_032,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin us,scenario_053,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this single-adult Idaho household with approximately $66,969 in annual income in 2026. The self-employment tax is derived from net self-employment income, which represents earnings from self-employment activities after allowable deductions. Since the computation trace shows only a zero value with no intermediate calculations, this indicates the household had no self-employment income to report for the tax year. Self-employment tax only applies when an individual has net earnings from self-employment exceeding $400, so the $0 result reflects that this household's income came from sources other than self-employment, such as wages or salaries.", us,scenario_091,federal_income_tax_before_refundable_credits,1349.998779296875,56,"PolicyEngine calculated the federal income tax before refundable credits as $1,350 for this Wisconsin single filer in 2026. The calculation began with gross income of $35,055.84, which included $27,000 in employment income, $4,806 in taxable interest income, and $4,176 in dividend income. After subtracting pre-tax 401(k) contributions of $926.16 and above-the-line deductions of $43.28, the adjusted gross income was $35,012.56. The taxable income was reduced to $18,284.66 by applying the standard deduction of $16,100 and an overtime income deduction of $627.91, then excluding $3,968 in qualified dividend income from taxation. Applying the 2026 tax brackets to this taxable income produced an income tax before credits of $1,470, which was then reduced by $120 in savers credit non-refundable credits, yielding the final value of $1,350.", us,scenario_111,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not WIC eligible (is_wic_eligible = False) for this Washington household in 2026. The WIC (Women, Infants, and Children) program is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this household contains only two adults with no children. Since the spouse does not fall into any of the categorical groups that qualify for WIC benefits, PolicyEngine correctly assigned an eligibility value of False, resulting in a reference value of 0.0 for spouse_wic_eligible.", -us,scenario_053,state_income_tax_before_refundable_credits,2435.2783203125,63,"PolicyEngine calculated Idaho state income tax before refundable credits at $2,435.28 for this single Idaho resident with approximately $66,969 in household income. The calculation began with an adjusted gross income (AGI) of $66,968.68, from which Idaho deductions of $16,100 were subtracted to arrive at taxable income of $50,868.68. The Idaho deductions included the standard deduction of $16,100 and itemized deductions of $377.35, which incorporated a state and local sales or income tax (SALT) deduction of $2,435.28. This taxable income of $50,868.68 was then subject to Idaho's tax brackets and rates, resulting in the final tax liability of $2,435.28 before any refundable credits.", +us,scenario_053,state_income_tax_before_refundable_credits,2186.05712890625,81,"PolicyEngine calculated Idaho income tax before refundable credits of $2,186.06 for this household in 2026: $2,176.06 of income tax plus the $10.00 permanent building fund tax. Federal AGI is $66,968.68 of employment income. Idaho subtracts $5,000.00 for the health insurance premiums the household pays, giving Idaho AGI of $61,968.68. The $16,100.00 standard deduction exceeds the $377.35 of itemized deductions (the medical expense deduction), and subtracting it leaves Idaho taxable income of $45,868.68. The tax on that income is $2,176.06, and no non-refundable credit reduces it.", us,scenario_000,head_medicare_eligible,1.0,1,"PolicyEngine determined that the household head is Medicare eligible (value = 1.0, or True) for the 2026 tax year. This determination is based on the household's demographic characteristics and age status as of the relevant tax year. The single adult in this Texas household meets the eligibility criteria established by Medicare rules, which primarily depend on age (typically 65 or older) and citizenship or permanent residency status. With a household income of approximately $111,569, income does not affect Medicare eligibility, as Medicare is a universal program for qualifying individuals regardless of earnings. Therefore, PolicyEngine assigned a value of True to indicate that the head of household qualifies for Medicare benefits in 2026.", us,scenario_091,federal_refundable_credits,0.0,6,"For this single adult in Wisconsin with approximately $35,982 in household income and no children, PolicyEngine calculated federal refundable credits of $0. The calculation examined five potential refundable credit components: the Earned Income Tax Credit (eitc), the refundable portion of the American Opportunity Credit (refundable_american_opportunity_credit), the refundable Child Tax Credit (refundable_ctc), the recovery rebate credit (recovery_rebate_credit), and the refundable payroll tax credit (refundable_payroll_tax_credit). Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. This outcome reflects that the household does not qualify for any of these refundable credits under 2026 tax law—most notably, the EITC requires either dependent children or to fall within specific age and income ranges for childless workers, and the refundable CTC requires dependent children, neither of which apply to this single adult household.", us,scenario_015,payroll_tax,3442.5,5,"PolicyEngine calculated a payroll tax of $3,442.50 for this single Indiana resident with approximately $43,450 in household income. The total payroll tax consists of two components: employee Social Security tax of $2,790 and employee Medicare tax of $652.50. The Social Security tax was computed by applying the 6.2% employee rate to earnings up to the 2026 Social Security wage base, while the Medicare tax was calculated by applying the 1.45% employee rate to total wages. Since this household member is identified as a tax unit head, they are subject to the standard employee payroll tax obligations without any special exemptions or adjustments.", @@ -1124,7 +1124,7 @@ us,scenario_033,federal_refundable_credits,0.0,6,"PolicyEngine calculated federa us,scenario_091,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Wisconsin household because the household has no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $35,982 in total income derives from other sources (such as wages or salaries), there is no self-employment income base upon which to apply the 15.3% self-employment tax rate. Consequently, PolicyEngine's computation trace shows self_employment_tax = 0, reflecting the absence of any self-employment activity subject to Social Security and Medicare taxes.", us,scenario_053,state_refundable_credits,155.0,76,"For this Idaho resident in 2026, PolicyEngine calculated total state refundable credits of $155. This amount comes entirely from Idaho's grocery credit (id_grocery_credit), which contributed the full $155. The grocery credit calculation determined that the household qualified for all 12 months of the tax year, with each month meeting the qualifying criteria. The base grocery credit amount (id_grocery_credit_base) was set at $155, and since the household qualified for the full year with no phase-out or reduction applied, this base amount became the final credit value. All other states' refundable credits returned zero, as the household resides in Idaho.", us,scenario_015,state_income_tax_before_refundable_credits,1236.3267822265625,57,"PolicyEngine calculated Indiana state income tax before refundable credits of $1,236.33 for this single filer in 2026. The calculation began with the household's adjusted gross income of $42,909.38, from which Indiana's base exemption of $1,000 was subtracted to arrive at an Indiana AGI of $41,909.38. This taxable income amount was then subject to Indiana's tax rate schedule, resulting in the final tax liability of $1,236.33. All other states' income tax calculations returned zero, confirming that only Indiana's tax applied to this resident.", -us,scenario_091,state_income_tax_before_refundable_credits,843.6614990234375,73,"PolicyEngine calculated Wisconsin state income tax before refundable credits of $843.66 for this single Wisconsin resident. The calculation began with federal gross income of $35,055.84, composed of $26,073.84 in wages (after $926.16 in pre-tax contributions), $4,806 in taxable interest, and $4,176 in dividends, while excluding $1,170 in capital gain distributions reported without Schedule D. After applying a $43.28 traditional IRA deduction, Wisconsin adjusted gross income reached $35,012.56. Wisconsin taxable income was then determined by subtracting the Wisconsin standard deduction of $12,067.66 and the personal exemption of $700, yielding taxable income of $22,244.90. Applying Wisconsin's tax brackets to this taxable income produced the final tax liability of $843.66 before any refundable credits.", +us,scenario_091,state_income_tax_before_refundable_credits,878.5157470703125,80,"PolicyEngine calculated Wisconsin income tax before refundable credits of $878.52 for this one-person household in 2026. IRS gross income of $36,225.84 comprises $26,073.84 of IRS employment income ($27,000.00 of wages less $926.16 of pre-tax contributions), $4,806.00 of taxable interest, $4,176.00 of dividends and $1,170.00 of capital gain distributions. A $43.28 traditional IRA deduction gives federal AGI of $36,182.56. Wisconsin subtracts $351.00, 30% of the capital gain distributions, giving Wisconsin AGI of $35,831.56. After the $12,074.61 standard deduction and the $700.00 exemption, Wisconsin taxable income is $23,056.95, and the tax on it is $878.52 with no credit to reduce it.", us,scenario_000,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $111,569 places it at 6.99 times the federal poverty guideline, which exceeds the 185% income threshold required for reduced-price meal eligibility. Since Texas does not have universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is assigned to the PAID school_meal_tier and receives no subsidy (school_meal_net_subsidy = 0). Consequently, the household must pay the full price for school meals.", us,scenario_053,local_income_tax,0.0,1,"For this single adult in Idaho with a household income of approximately $66,969 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that Idaho does not impose a local income tax at the state level, and the household's residence does not appear to be subject to any municipal or county-level income taxes that would generate a tax liability. Consequently, despite the household's substantial income, no local income tax obligation was derived from PolicyEngine's computation.", us,scenario_033,payroll_tax,6043.5,5,"PolicyEngine calculated a total payroll tax of $6,043.50 for this South Dakota joint household by summing the employee_social_security_tax and employee_medicare_tax components. The employee social security tax totaled $4,898, split between the two adults as $1,612 and $3,286 respectively, reflecting their individual wage earnings. The employee medicare tax component added $1,145.50, distributed as $377 for one spouse and $768.50 for the other, proportional to their respective incomes. These two payroll tax components—social security and medicare—were combined to arrive at the final spm_unit_payroll_tax value of $6,043.50 for the tax year 2026.", @@ -1663,7 +1663,7 @@ us,scenario_057,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin us,scenario_022,head_medicare_eligible,1.0,1,"PolicyEngine determined that the household head is Medicare eligible (value = 1.0, or True) for the 2026 tax year. The calculation of `is_medicare_eligible` is based on age criteria established by the Social Security Administration, which sets Medicare eligibility at age 65 or older. Since the benchmark household head meets or exceeds this age threshold, PolicyEngine assigned a value of True to indicate Medicare eligibility status. This binary determination directly reflects federal Medicare program rules and does not depend on the household's income level of approximately $112,259 or other household characteristics.", us,scenario_001,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on age criteria. The head is 71 years old, which exceeds CHIP's maximum age limit for coverage eligibility. CHIP is designed to provide health insurance to children and pregnant women, with strict age restrictions that exclude adults over a certain threshold. Since the head of household fails CHIP's age requirement, they are ineligible for the program regardless of other factors such as income or Medicaid status.", us,scenario_117,child3_wic_eligible,0.0,1,"PolicyEngine determined that the third child in this Arkansas household is not eligible for WIC benefits, resulting in a value of False. WIC eligibility is primarily determined by income thresholds, which are set at 185% of the federal poverty line. With a household income of approximately $250,733 annually, this family's income substantially exceeds the WIC income limit for a household of five people. Since the household income far surpasses the maximum allowable threshold for WIC participation, none of the children in the household—including the third child—qualify for the program, regardless of other potential eligibility factors such as age or nutritional risk status.", -us,scenario_099,state_income_tax_before_refundable_credits,4493.7431640625,77,"PolicyEngine calculated California state income tax before refundable credits of $4,493.74 for this joint-filing household with two dependent children and ~$159,676 in household income. The calculation began with adjusted gross income of $159,585.05, from which California deductions of $20,476.57 were subtracted—comprising a standard deduction of $11,412 and itemized deductions of $20,476.57 (which included $11,823.08 in charitable contributions and $8,653.50 in real estate taxes). This produced California taxable income of $139,108.47, which was subject to California's tax brackets to yield a tax before credits of $5,778.21. Finally, PolicyEngine applied non-refundable credits of $1,284.46 (California exemptions based on the two dependent children) to arrive at the final state income tax before refundable credits of $4,493.74.", +us,scenario_099,state_income_tax_before_refundable_credits,4615.48388671875,91,"PolicyEngine calculated California income tax before refundable credits of $4,615.48 for this joint filer household in 2026. Federal AGI is $159,729.30. California does not allow the federal $337.50 educator expense deduction, so it adds it back, giving California AGI of $160,066.80. California itemized deductions of $21,274.50 are $8,653.50 of real estate taxes and a $12,621.00 charitable deduction, all of the household's cash ($11,932.94) and non-cash ($688.06) donations. They exceed the $11,412.00 standard deduction. Subtracting them leaves California taxable income of $138,792.30, and the tax on it is $5,871.48. Exemption credits of $1,256.00 for the joint filers and their two dependents reduce it to $4,615.48.", us,scenario_039,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Virginia household in 2026. The household, consisting of a single adult with approximately $84,625 in annual income, exceeds the income eligibility threshold for SNAP benefits. Although the model determined that the household would take up SNAP if eligible (takes_up_snap_if_eligible = True), the household's income level disqualifies it from receiving any benefits under the program's income limits. Therefore, despite the willingness to participate in the program, no SNAP assistance was awarded.", us,scenario_078,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Maryland household in 2026. The household's annual income of approximately $200,137 substantially exceeds the income eligibility thresholds for TANF, which are designed to assist low-income families. Additionally, with zero children in the household, the applicant does not meet the family composition requirements for TANF eligibility, as the program is primarily structured to provide cash assistance to families with dependent children. Consequently, PolicyEngine determined no TANF benefit applies to this household.", us,scenario_057,free_school_meals_eligible,0.0,317,"PolicyEngine determined that this Louisiana household of two adults with approximately $38,000 in annual income is **not eligible** for free school meals, resulting in a value of 0.0. Although the household meets the categorical eligibility requirement through SNAP participation (with a monthly allotment of $219), the school meal tier calculation returned a value of 0, which overrides eligibility. The household's countable income of $17,631 represents 81% of the federal poverty guideline (school_meal_fpg_ratio of 0.81), placing it below the 130% income threshold that would normally qualify for free meals. However, the school_meal_tier variable—which aggregates both categorical eligibility and income-based criteria—evaluated to 0, indicating that despite meeting categorical eligibility through SNAP, some other factor in the tier determination resulted in ineligibility. Consequently, the school_meal_net_subsidy is $0, meaning no federal subsidy applies to this household's school meals.", diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json index c8706075..df1db9c6 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json @@ -1,5 +1,5 @@ { - "note": "Wording-only amendments of published text for cases GPT-6.1 Sol re-opened (finish_gpt61sol.py load_amendments): each replaces its old text, found exactly once, with the new text. None changes a class, an exclusion or a score. Triage applies them; export binds this file; the freeze commits it beside the adjudication record.", + "note": "Wording-only amendments of published text for cases GPT-6.1 Sol re-opened (finish_gpt61sol.py load_amendments): each replaces its old text, found exactly once, with the new text. None changes a class, an exclusion or a score. Triage applies them; export binds this file; the freeze commits it beside the adjudication record. The last 462 (after the first 394) are for cases Muse Wonder re-opened (finish_musewonder.py load_amendments). Wording amendments to the cases the Muse Wonder release re-opened (dashboard-data-20261013). These cases' new Opus 5.5 verdicts replaced text earlier releases had corrected (us_wording-amendments.json, docs/release_20261006/annotation_rewrites.json and reference_audit/2026-10-05-medicaid-031-annotations/rewrites.json). A GPT-6.1 Sol reviewer read each of the 26 cases' new text against the record and proposed corrections; a second GPT-6.1 Sol reviewer checked each one adversarially, keeping 444, refining the wording of 20 and dropping 80; 2 more that would have rewritten the generated adjudication sentence, which restates the record, were left out. Each corrects a statement the record contradicts; none changes a score or a decision.", "amendments": [ { "case_id": "us__scenario_008__state_refundable_credits", @@ -3122,6 +3122,3682 @@ "old": "which yields $461/year.", "new": "which yields the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The unsupported income-and-resources claim stays." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The closest models got the 2026 brackets and NIIT right but missed the non-itemizer charitable deduction and the 401(k), state-refund, and IRA adjustments to AGI. The far misses assumed the TCJA expired (Pease, AMT, uncapped SALT), invented mortgage interest, subtracted employer premiums from wages, or left out NIIT.", + "new": "The closest models differed from the frozen reference in the non-itemizer charitable deduction, 401(k) allocation, refund treatment, or IRA deduction. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g); the frozen reference improperly allows it. The record leaves refund taxation and excess 401(k) allocation unresolved. The far misses assumed the TCJA expired (Pease, AMT, uncapped SALT), invented mortgage interest, subtracted employer premiums from wages, or omitted NIIT included in the frozen reference; whether this output includes NIIT also remains unresolved.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g and r25_niit_in_federal_output), adjudication.json and reference_outputs.csv.meta.json revisions[17]–[19].excluded_outputs_rechecked identify the frozen IRA deduction as defective and leave refund taxation, excess-deferral allocation and NIIT inclusion unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "which belongs in this line", + "new": "which the frozen reference includes although the record leaves its inclusion in this output unresolved", + "reason": "reference_exclusions.json names r25_niit_in_federal_output as an unlisted-input issue, and reference_outputs.csv.meta.json revisions[18]–[19].excluded_outputs_rechecked retain the exclusion because the newer engine does not settle NIIT inclusion." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "Its brackets, 15% dividend tax, and $5,722 NIIT match the derivation, but taxable income is $559 too high. It skipped the $1,070 non-itemizer cash charitable deduction and counted the $4,148 state refund as income. It also deducted the full $46,308 of traditional 401(k) deferrals instead of $41,650, the amount left after splitting the $24,500 elective-deferral limit pro rata across each spouse's $27,240 of combined Roth and traditional deferrals.", + "new": "Its brackets, 15% dividend tax, and $5,722 NIIT match the frozen derivation, but taxable income is $559 above the frozen reference. Differences include the $1,070 non-itemizer charitable deduction, refund treatment, and the frozen engine's pro rata 401(k) allocation of $41,650 rather than $46,308; refund taxation and excess-deferral allocation are unresolved. Both the model and the frozen reference deduct $2,164 of IRA contributions, which the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked establish that the frozen IRA deduction is defective and that refund taxation and excess 401(k) allocation are unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct path is the $32,200 standard deduction plus the $1,070 non-itemizer deduction, giving $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax.", + "new": "The frozen reference uses the $32,200 standard deduction plus the $1,070 non-itemizer deduction, giving $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax; those figures include a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json distinguish the defective frozen $106,505.90 from the adjudicated corrected $107,198.34, which allows no IRA deduction." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "The correct computation gives $89,096 of ordinary tax on $425,575, $11,688 on $77,920 of qualified dividends, and $5,722 of NIIT.", + "new": "The frozen computation gives $89,096 of ordinary tax on $425,575, $11,688 on $77,920 of qualified dividends, and $5,722 of NIIT; its taxable income includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying these frozen figures as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "Explicitly left out NIIT ('NIIT not part of this line'), which adds $5,722. It then reported $91,458, roughly its ordinary-bracket tax alone, dropping the $11,688 qualified-dividend tax it had computed itself (its own total was $102,907). It also used a $31,500 standard deduction, skipped the $1,070 non-itemizer charitable deduction, and counted the $4,148 state refund as income.", + "new": "Explicitly left out the $5,722 NIIT included in the frozen reference ('NIIT not part of this line'); the record leaves NIIT inclusion unresolved. It then reported $91,458, roughly its ordinary-bracket tax alone, dropping the $11,688 qualified-dividend tax it had computed itself (its own total was $102,907). It also used a $31,500 standard deduction and skipped the $1,070 non-itemizer charitable deduction. Its inclusion of the $4,148 refund differs from the frozen reference, but refund taxation is unresolved.", + "reason": "reference_exclusions.json names NIIT inclusion as an unlisted-input issue, while reference_outputs.csv.meta.json revisions[17]–[19].excluded_outputs_rechecked record alternative refund treatments and retain the NIIT-related exclusion." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "The correct AGI is $536,764, followed by the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction.", + "new": "The frozen reference's AGI is $536,764, followed by the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction; that AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked establish that the frozen AGI improperly includes the IRA deduction." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "Its brackets, 15% dividend tax, and $5,722 NIIT match the derivation, but taxable income ($504,054) is $559 too high. It skipped the $1,070 non-itemizer cash charitable deduction and counted the $4,148 state refund as income. It also deducted the full $46,308 of 401(k) deferrals instead of $41,650, the amount left after splitting the $24,500 elective-deferral limit pro rata across each spouse's combined Roth and traditional deferrals.", + "new": "Its brackets, 15% dividend tax, and $5,722 NIIT match the frozen derivation, but taxable income ($504,054) is $559 above the frozen reference. Differences include the $1,070 non-itemizer charitable deduction, refund treatment, and the frozen engine's pro rata 401(k) allocation of $41,650 rather than $46,308; refund taxation and excess-deferral allocation are unresolved. Both the model and the frozen reference deduct $2,164 of IRA contributions, which the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked establish that the frozen IRA deduction is defective and that refund taxation and excess 401(k) allocation are unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "With no mortgage interest listed, the household takes the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction, giving $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax.", + "new": "With no mortgage interest listed, the household takes the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. The frozen reference reports $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax, including a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen taxable income and tax as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It also counted the $4,148 state refund, denied the $2,164 IRA deduction, and deducted the full $46,308 of 401(k) deferrals instead of the pro-rated $41,650.", + "new": "Its refund inclusion and $46,308 of 401(k) deferrals differ from the frozen reference's refund exclusion and pro rata $41,650 allocation; the record leaves those treatments unresolved. Denying the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g), whereas the frozen reference improperly allows it.", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and identify refund taxation and excess-deferral allocation as unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "Used a $30,750 standard deduction instead of $32,200, skipped the $1,070 non-itemizer charitable deduction, and counted the $4,148 state refund, which put taxable income at $507,668 instead of $503,495.", + "new": "Used a $30,750 standard deduction instead of $32,200 and skipped the $1,070 non-itemizer charitable deduction. Its $507,668 taxable income exceeds the frozen reference's $503,495, which includes a defective $2,164 IRA deduction disallowed under 219(g); its refund inclusion also differs from the frozen reference, but refund taxation is unresolved.", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the frozen IRA deduction as defective and record alternative refund treatments." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "2026 law gives the $32,200 standard deduction plus the $1,070 non-itemizer deduction and $89,096 of ordinary tax on $425,575.", + "new": "2026 law gives the $32,200 standard deduction plus the $1,070 non-itemizer deduction. The frozen reference's $89,096 of ordinary tax on $425,575 also reflects a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen ordinary tax as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "The correct AGI is $536,764, leading to $89,096 + $11,688 + $5,722.", + "new": "The frozen reference's AGI is $536,764, leading to $89,096 + $11,688 + $5,722; it includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the IRA deduction underlying the frozen AGI as erroneous." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "Subtracted the $13,369 of employer-sponsored insurance premiums from wages, giving AGI of $525,049; wages are reduced only by the $41,650 of allowed 401(k) deferrals, for AGI of $536,764.", + "new": "Subtracted the $13,369 of employer-sponsored insurance premiums from wages, giving AGI of $525,049, compared with the frozen reference's $536,764. The frozen engine uses a pro rata $41,650 traditional 401(k) allocation and improperly deducts $2,164 of IRA contributions; the record leaves excess-deferral allocation unresolved and disallows the IRA deduction under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the frozen IRA deduction as defective and the excess 401(k) allocation as unlisted." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "2026 law keeps the TCJA brackets, and this household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction, for $89,096 of ordinary tax, $11,688 of dividend tax, and $5,722 of NIIT.", + "new": "2026 law keeps the TCJA brackets, and this household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction. The frozen reference's $89,096 ordinary tax, $11,688 dividend tax, and $5,722 NIIT also reflect a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen tax components as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "Gave no derivation. Its $106,093 is $413 below the correct figure, which matches the right structure (2026 brackets, 15% on $77,920, NIIT) applied to ordinary taxable income about $1,290 too low. The correct taxable income is $503,495: wages net of only $41,650 of 401(k) deferrals, the $2,164 IRA deduction, and $33,270 of standard plus non-itemizer charitable deductions.", + "new": "Gave no derivation. Its $106,093 is $413 below the frozen reference, consistent with the frozen tax structure applied to ordinary taxable income about $1,290 below the frozen amount. The frozen reference's $503,495 taxable income uses the engine's pro rata $41,650 traditional 401(k) allocation, $33,270 of standard plus charitable deductions, and a defective $2,164 IRA deduction disallowed under 219(g); excess-deferral allocation is unresolved.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the frozen taxable income as defective and excess 401(k) allocation as unlisted." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "AGI $525,049 instead of $536,764", + "new": "AGI $525,049, compared with the frozen reference's $536,764, which includes a defective $2,164 IRA deduction disallowed under 219(g)", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the IRA deduction underlying the frozen AGI as erroneous." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "The correct computation taxes $425,575 of ordinary income at 2026 TCJA-structure brackets ($89,096) and adds $11,688 of dividend tax and $5,722 of NIIT.", + "new": "The frozen computation taxes $425,575 of ordinary income at 2026 TCJA-structure brackets ($89,096) and adds $11,688 of dividend tax and $5,722 of NIIT; its taxable income includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen tax components as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "Its $534,270 AGI deducted the full $46,308 of 401(k) deferrals rather than the pro-rated $41,650 and left out the $2,164 IRA deduction. It then rounded to $105,000 instead of computing $89,096 + $11,688 + $5,722.", + "new": "Its $534,270 AGI uses $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation, which the prompt does not settle. It correctly omits the $2,164 IRA deduction disallowed under 219(g). Its rounded $105,000 differs from the frozen reference's $89,096 + $11,688 + $5,722, which includes that defective IRA deduction.", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and identify the excess 401(k) allocation as unlisted." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "It deducted the full $46,308 of 401(k) deferrals instead of the pro-rated $41,650 and skipped the $2,164 IRA deduction.", + "new": "It deducted $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation, which the prompt does not settle. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g); the frozen reference improperly allows it.", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and identify excess 401(k) allocation as unlisted." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Reported $21,735 with no bracket work, about one-fifth of the actual liability. Taxable income of $503,495 produces $89,096 of ordinary tax, $11,688 on qualified dividends, and $5,722 of NIIT.", + "new": "Reported $21,735 with no bracket work, about one-fifth of the frozen reference. The frozen $503,495 taxable income produces $89,096 ordinary tax, $11,688 dividend tax, and $5,722 NIIT, but includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen liability as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "The correct computation taxes $425,575 of ordinary income at 2026 joint brackets ($89,096) and $77,920 of qualified dividends at 15% ($11,688), then adds $5,722 of NIIT.", + "new": "The frozen computation taxes $425,575 of ordinary income at 2026 joint brackets ($89,096) and $77,920 of qualified dividends at 15% ($11,688), then adds $5,722 of NIIT; its taxable income includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen tax components as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "Used the $32,200 standard deduction plus the $1,070 non-itemizer deduction and the $5,722 NIIT, but its $538,418 AGI is $1,654 too high. It counted the $4,148 state refund and left out the $2,164 IRA deduction. That was partly offset by deducting the full $46,308 of 401(k) deferrals instead of the $41,650 allowed after splitting the $24,500 elective-deferral limit across each spouse's combined Roth and traditional deferrals.", + "new": "Used the $32,200 standard deduction plus the $1,070 non-itemizer deduction and the $5,722 NIIT, but its $538,418 AGI is $1,654 above the frozen reference's. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen refund exclusion and pro rata $41,650 allocation; those treatments are unresolved. Its answer is $163.42 below the adjudicated corrected $107,198.34.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked distinguish the defective frozen value from the corrected value and leave refund taxation and excess-deferral allocation unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "Gave no numbers in its reasoning. Its $104,535 is $1,971 below the correct figure, matching ordinary taxable income about $6,160 lower than the correct $425,575. That ordinary income is taxed at 2026 brackets ($89,096), plus $11,688 on qualified dividends and $5,722 of NIIT.", + "new": "Gave no numbers in its reasoning. Its $104,535 is $1,971 below the frozen reference, matching ordinary taxable income about $6,160 below the frozen $425,575. The frozen components are $89,096 ordinary tax, $11,688 dividend tax, and $5,722 NIIT, but include a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen figures as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "Counted the $4,148 state refund as taxable and itemized instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer cash charitable deduction. That left taxable income about $2,000 above the correct $503,495 and tax $632 too high.", + "new": "Included the $4,148 refund and itemized, differing from the frozen reference's refund exclusion and $32,200 standard deduction plus $1,070 charitable deduction; refund taxation is unresolved. Its tax is $632 above the frozen reference, whose $503,495 taxable income includes a defective $2,164 IRA deduction disallowed under 219(g). Its answer is $60.34 below the adjudicated corrected $107,198.34.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked distinguish the defective frozen value from the corrected value and record alternative refund treatments." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "Its total matches taxable income of $506,218, the $32,200 standard deduction without the $1,070 non-itemizer cash charitable deduction. It also used AGI of $538,418, which counts the $4,148 refund, omits the $2,164 IRA deduction, and deducts the full $46,308 of 401(k) deferrals instead of $41,650. That overstates tax by $871.", + "new": "Its total matches taxable income of $506,218 with the $32,200 standard deduction and no $1,070 non-itemizer charitable deduction. Its inferred $538,418 AGI includes the refund and uses $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its tax is $871 above the frozen reference, but only $178.66 above the adjudicated corrected $107,198.34.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and distinguish frozen comparisons from corrected treatment while leaving refund taxation and excess-deferral allocation unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "Subtracted $13,369 of employer-sponsored insurance premiums from wages (AGI $525,049 instead of $536,764) and itemized $36,959, so taxable income came out too low. Wages are reduced only by the $41,650 of allowed 401(k) deferrals, and the deduction is the $32,200 standard plus the $1,070 non-itemizer charitable deduction.", + "new": "Subtracted $13,369 of employer-sponsored insurance premiums from wages (AGI $525,049 compared with the frozen reference's $536,764) and itemized $36,959, producing taxable income below the frozen reference's. The frozen engine uses a pro rata $41,650 traditional 401(k) allocation and $32,200 standard plus $1,070 charitable deductions, but also improperly deducts $2,164 of IRA contributions disallowed under 219(g); excess-deferral allocation is unresolved.", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the frozen IRA deduction as defective and the excess 401(k) allocation as unlisted." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "Counted the $4,148 state refund in AGI and left out the $2,164 IRA deduction, while deducting the full $46,308 of 401(k) deferrals instead of the pro-rated $41,650. That overstated taxable income by about $1,800 and tax by $571.", + "new": "Included the $4,148 refund and deducted $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its tax is $571 above the frozen reference, but $121.34 below the adjudicated corrected $107,198.34.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and distinguish the defective frozen value from corrected treatment while leaving refund taxation and excess-deferral allocation unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "Subtracted the $13,369 of employer insurance premiums from wages (AGI $525,049 instead of $536,764) and itemized a phased-down SALT deduction instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Tax came out $4,106 too low.", + "new": "Subtracted the $13,369 of employer insurance premiums from wages (AGI $525,049 compared with the frozen reference's $536,764) and itemized a phased-down SALT deduction instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Its tax is $4,106 below the frozen reference, whose AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen AGI and tax as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "Subtracted the $13,369 of employer insurance premiums from wages (AGI $525,049 instead of $536,764) and used $34,385 of itemized deductions instead of the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Tax came out $4,106 too low.", + "new": "Subtracted the $13,369 of employer insurance premiums from wages (AGI $525,049 compared with the frozen reference's $536,764) and used $34,385 of itemized deductions instead of the $32,200 standard deduction plus the $1,070 non-itemizer charitable deduction. Its tax is $4,106 below the frozen reference, whose AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen AGI and tax as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "and reported $112,450 without a derivation, about $5,944 too high. The correct total is $89,096 + $11,688 + $5,722 after the $32,200 standard deduction plus the $1,070 non-itemizer deduction.", + "new": "and reported $112,450 without a derivation, about $5,944 above the frozen reference. The frozen components are $89,096 + $11,688 + $5,722 after the $32,200 standard deduction plus the $1,070 non-itemizer deduction, but also reflect a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen total as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "In 2026 the household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction and owes no AMT, so the total is $89,096 + $11,688 + $5,722.", + "new": "In 2026 the household takes the $32,200 standard deduction plus the $1,070 non-itemizer deduction and owes no AMT. The frozen total of $89,096 + $11,688 + $5,722 also reflects a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen total as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "Its $538,418 AGI counts the $4,148 refund, omits the $2,164 IRA deduction, and deducts the full $46,308 of 401(k) deferrals instead of $41,650. That left taxable income at $506,918 instead of $503,495.", + "new": "Its $538,418 AGI includes the refund and deducts $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its $506,918 taxable income exceeds the defective frozen $503,495, and its answer is $332.66 above the adjudicated corrected $107,198.34.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and leave refund taxation and excess-deferral allocation unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "It also subtracted health premiums from wages, giving AGI of $525,049 instead of $536,764.", + "new": "It also subtracted health premiums from wages, giving AGI of $525,049 compared with the frozen reference's $536,764, which includes a defective $2,164 IRA deduction disallowed under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked identify the IRA deduction underlying the frozen AGI as erroneous." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "Its $538,418 AGI counts the $4,148 state refund, omits the $2,164 IRA deduction, and deducts the full $46,308 of 401(k) deferrals instead of the pro-rated $41,650. Taxable income came out at $506,218 instead of $503,495, overstating tax by $871.", + "new": "Its $538,418 AGI includes the refund and deducts $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g). Its $506,218 taxable income exceeds the defective frozen $503,495; its tax is $871 above the frozen reference, but only $178.98 above the adjudicated corrected $107,198.34.", + "reason": "reference_exclusions.json (r02_ira_219g), adjudication.json and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and distinguish frozen comparisons from corrected treatment while leaving refund taxation and excess-deferral allocation unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "With $536,764 of AGI and $33,270 of standard plus non-itemizer deductions, tax is $106,506.", + "new": "The frozen reference reports $536,764 of AGI, $33,270 of standard plus non-itemizer deductions, and $106,506 of tax; those figures include a defective $2,164 IRA deduction that the adjudicated corrected $107,198.34 disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json distinguish the defective frozen total from the corrected value, which allows no IRA deduction." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "Subtracted employer insurance premiums from wages, putting AGI near $525,000 instead of $536,764, and gave no derivation for the remaining steps. The lower AGI understated tax by $2,806.", + "new": "Subtracted employer insurance premiums from wages, putting AGI near $525,000 compared with the frozen reference's $536,764, and gave no derivation for the remaining steps. Its tax is $2,806 below the frozen reference, whose AGI includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) and adjudication.json identify the IRA deduction underlying the frozen AGI and tax as an engine defect." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "It used AGI of $538,418, which counts the $4,148 refund, omits the $2,164 IRA deduction, and deducts the full $46,308 of 401(k) deferrals instead of $41,650.", + "new": "Its $538,418 AGI includes the refund and deducts $46,308 of traditional 401(k) deferrals rather than the frozen engine's pro rata $41,650 allocation; those treatments are unresolved. Omitting the $2,164 IRA deduction agrees with the exclusion's corrected treatment under 219(g), whereas the frozen reference improperly allows it.", + "reason": "reference_exclusions.json (r02_ira_219g) and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked require no IRA deduction and leave refund taxation and excess-deferral allocation unresolved." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "It added an AMT that does not arise and left out the $5,722 NIIT, so its $89,067 understates the $106,506 liability.", + "new": "It added an AMT that does not arise and omitted the $5,722 NIIT included in the frozen reference; NIIT inclusion in this output is unresolved. Its $89,067 is below the frozen $106,506, which includes a defective $2,164 IRA deduction that the exclusion's corrected treatment disallows under 219(g).", + "reason": "reference_exclusions.json (r02_ira_219g and r25_niit_in_federal_output), adjudication.json and reference_outputs.csv.meta.json revisions[18]–[19].excluded_outputs_rechecked identify the frozen IRA deduction as defective and NIIT inclusion as unresolved." + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It also held a prior-year bracket schedule instead of 2026 indexing and took a $222 exemption credit instead of $114.", + "new": "It used the 2025 bracket schedule and exemption amounts that the release's California convention (c_ca_hold_2025) holds for 2026; its $222 exemption credit reflects its understated AGI, compared with $114 in the current reference.", + "reason": "reference_outputs.csv.meta.json revisions[1], [18] and [19], together with latest_c_ca_hold_2025.py, establish that the release holds California's published 2025 brackets and exemption amounts for 2026, as the prior amendments already explained." + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It held a prior-year bracket schedule constant instead of 2026 indexing and took a $222 exemption credit instead of $114.", + "new": "It used the 2025 bracket schedule and exemption amounts that the release's California convention (c_ca_hold_2025) holds for 2026; its $222 exemption credit reflects its understated AGI, compared with $114 in the current reference.", + "reason": "reference_outputs.csv.meta.json revisions[1], [18] and [19], together with latest_c_ca_hold_2025.py, establish that the release holds California's published 2025 brackets and exemption amounts for 2026, as the prior amendments already explained." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "which every Idaho filer owes", + "new": "which this household owes", + "reason": "reference_outputs.csv.meta.json revisions[18] records the $10 building fund tax as conditional on id_pbf_liable, while prompt.md confirms this household owes it." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "That sales-tax amount plus the $13,834 of real estate taxes comes to $17,090.34. This is under the $40,400 cap and above the $16,100 standard deduction, so the filer itemizes. Because the filer itemizes, the $196 gift falls below the 0.5%-of-AGI charitable floor and the non-itemizer charitable deduction doesn't apply.", + "new": "The frozen reference projects the sales-tax amount from the IRS's 2023 table, making SALT $17,090.34 and causing it to itemize and disallow the $196 gift under the 0.5%-of-AGI floor. The release's sales tax convention (c_irs_sales_tax_2025) instead uses the IRS 2025 Texas amount of $1,595, leaving SALT below the $16,100 standard deduction, so the filer takes the standard deduction plus the $196 non-itemizer charitable deduction.", + "reason": "reference_outputs.csv.meta.json revisions[2] establishes c_irs_sales_tax_2025, and revisions[17]'s scenario_020 recheck gives the IRS 2025 Texas size-1 $300k+ cell as $1,595, leaving SALT below the standard deduction." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Taxable income is $300,150.19 AGI (after the $5,708.82 alimony deduction) minus $17,090.34, which is $283,059.84. Regular tax of $67,834.07 plus $222.64 of NIIT gives $68,056.71.", + "new": "Under that convention, taxable income is $300,150.19 AGI (after the $5,708.82 alimony deduction) minus $16,296, or $283,854.19; regular tax is $68,112.09, and including the $222.64 NIIT gives $68,334.73. The excluded frozen reference instead uses taxable income of $283,059.84 and regular tax of $67,834.07 plus NIIT, giving $68,056.71.", + "reason": "reference_outputs.csv.meta.json revisions[17] and revisions[19] recheck scenario_020 at $68,334.73 with NIIT, while reference_exclusions.json records $68,112.09 with every convention and NIIT excluded, corresponding to the $16,100 standard and $196 non-itemizer charitable deductions." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The models near the reference all took the standard deduction, which leaves taxable income $794–$990 too high.", + "new": "The models near the reference took the standard deduction, as the convention does; those that also took the $196 non-itemizer charitable deduction land within $0.14 of $68,334.73 with NIIT or $68,112.09 without it.", + "reason": "reference_outputs.csv.meta.json revisions[17] makes the standard deduction appropriate, and prompt.md records the nearby answers within $0.14 of its $68,334.73 recheck with NIIT or reference_exclusions.json's $68,112.09 alternative without NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "Counted only the $13,834 of real estate taxes as SALT and missed the general sales tax deduction available to Texas residents, which raises itemized deductions to $17,090.34, above the $16,100 standard deduction. It took the standard deduction and got $284,050 of taxable income instead of $283,059.84, so its tax is about $347 too high.", + "new": "The standard deduction is the household's choice under c_irs_sales_tax_2025, whose $1,595 IRS 2025 sales tax amount leaves SALT below $16,100. It omitted the $196 non-itemizer charitable deduction, giving taxable income of $284,050 instead of $283,854.19 and tax of $68,404 instead of $68,334.73 with NIIT.", + "reason": "prompt.md records this answer, and reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 under c_irs_sales_tax_2025 at $68,334.73 with NIIT using the standard deduction plus $196 of non-itemizer charity." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "Missed the general sales tax deduction that Texas residents add to the $13,834 of real estate taxes. That makes itemized SALT $17,090.34 and beats the $16,100 standard deduction. It instead used the standard deduction plus the $196 non-itemizer charitable deduction, giving $283,854 of taxable income instead of $283,059.84.", + "new": "Used the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, as appropriate under c_irs_sales_tax_2025 because the IRS 2025 sales tax amount leaves SALT below the standard deduction. Its $68,334.59 is $0.14 below the conventions' $68,334.73 with NIIT.", + "reason": "prompt.md records this answer, and reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 under c_irs_sales_tax_2025 at $68,334.73 with NIIT using the standard deduction plus $196 of non-itemizer charity." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "It also used a $15,000 standard deduction and missed the sales-tax-plus-property-tax itemized deduction of $17,090.34.", + "new": "It also used a $15,000 standard deduction instead of the 2026 $16,100 standard deduction plus the $196 non-itemizer charitable deduction appropriate under c_irs_sales_tax_2025.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "The correct calculation is $300,150.19 AGI minus $17,090.34 of itemized SALT, giving $283,059.84 of taxable income and $68,056.71 including NIIT.", + "new": "Under c_irs_sales_tax_2025, $300,150.19 AGI minus the $16,100 standard deduction and $196 non-itemizer charitable deduction gives $283,854.19 of taxable income and $68,334.73 with NIIT, or the exclusion record's $68,112.09 without it.", + "reason": "reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 at $68,334.73 with NIIT under c_irs_sales_tax_2025, while reference_exclusions.json records $68,112.09 without NIIT and keeps the frozen reference excluded." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "Guessed itemized deductions of $30,000–$40,000 with no listed basis, when the correct figure is $17,090.34 (real estate taxes plus the Texas general sales tax deduction). It then added an unexplained overtime adjustment to reach $72,700 instead of $68,056.71.", + "new": "Guessed itemized deductions of $30,000–$40,000 with no listed basis; c_irs_sales_tax_2025 leaves SALT below the $16,100 standard deduction, which combines with the $196 non-itemizer charitable deduction. Its final $72,700 estimate is unexplained; the convention result is $68,334.73 with NIIT, or $68,112.09 without it.", + "reason": "reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 at $68,334.73 with NIIT under c_irs_sales_tax_2025, while reference_exclusions.json records $68,112.09 without NIIT and keeps the frozen reference excluded." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "Treated SALT as only the $13,834 of real estate taxes and missed the general sales tax deduction for a no-income-tax state like Texas, which brings itemized deductions to $17,090.34, above the $16,100 standard deduction. Taking the standard deduction plus the $196 non-itemizer charitable deduction left taxable income $794 too high, at $283,854.", + "new": "Taking the $16,100 standard deduction plus the $196 non-itemizer charitable deduction agrees with c_irs_sales_tax_2025, whose IRS 2025 sales tax amount leaves SALT below the standard deduction. Its $68,334.59 is $0.14 below the conventions' $68,334.73 with NIIT.", + "reason": "prompt.md records this answer, and reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 under c_irs_sales_tax_2025 at $68,334.73 with NIIT using the standard deduction plus $196 of non-itemizer charity." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It missed the Texas general sales tax deduction that makes itemizing worth $17,090.34, and it used projected brackets and a $15,400 standard deduction instead of the 2026 parameters.", + "new": "Under c_irs_sales_tax_2025, the IRS 2025 sales tax amount leaves SALT below the standard deduction; it used projected brackets and a $15,400 standard deduction instead of the 2026 parameters.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It also missed the general sales tax deduction that raises Texas SALT to $17,090.34, above the standard deduction, so taxable income came out at $289,759 instead of $283,059.84.", + "new": "It also omitted the $196 non-itemizer charitable deduction from its standard deduction, giving taxable income of $289,759 instead of $283,854.19 under c_irs_sales_tax_2025.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "instead of $283,059.84.", + "new": "instead of $283,854.19 under c_irs_sales_tax_2025.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "took the standard deduction instead of $17,090.34 of itemized SALT,", + "new": "used a $15,350 standard deduction instead of the $16,100 standard deduction plus the $196 non-itemizer charitable deduction appropriate under c_irs_sales_tax_2025,", + "reason": "prompt.md records this model's $15,350 standard deduction, while c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and revisions[17] makes the $16,100 standard deduction plus $196 of non-itemizer charity the household's choice." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "The correct regular-tax path is itemized SALT of $17,090.34, giving $283,059.84 of taxable income.", + "new": "The correct regular-tax path under c_irs_sales_tax_2025 uses the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, giving $283,854.19 of taxable income.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "Denied the alimony deduction, giving AGI of $305,859, and capped SALT at $10,000 instead of $40,400, so it missed the $17,090.34 SALT itemization (real estate taxes plus the Texas sales tax deduction). That deduction exceeds the standard deduction.", + "new": "Denied the alimony deduction, giving AGI of $305,859, and capped SALT at $10,000 instead of $40,400. Under c_irs_sales_tax_2025, SALT remains below the standard deduction, so the household uses the $16,100 standard deduction plus the $196 non-itemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "The correct itemized total is $17,090.34 of SALT.", + "new": "Under c_irs_sales_tax_2025, SALT remains below the $16,100 standard deduction, so the household uses that deduction plus the $196 non-itemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It missed the Texas sales tax itemization of $17,090.34 and left out the $222.64 of NIIT.", + "new": "Under c_irs_sales_tax_2025, the household uses the $16,100 standard deduction plus the $196 non-itemizer charitable deduction; it left out the $222.64 NIIT included in the frozen reference.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "It counted only $13,834 of property tax as SALT and missed the general sales tax deduction that raises Texas itemized deductions to $17,090.34, above the $16,100 standard deduction.", + "new": "Its standard deduction is appropriate under c_irs_sales_tax_2025, whose IRS 2025 table leaves SALT below $16,100; it omitted the $196 non-itemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Correct itemized deductions are only $17,090.34 of SALT, leaving $283,059.84 of taxable income.", + "new": "Under c_irs_sales_tax_2025, SALT stays below the $16,100 standard deduction, and that deduction plus $196 of non-itemizer charity leaves $283,854.19 of taxable income.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "Took the $16,100 standard deduction (taxable income $284,050) and missed the general sales tax deduction for Texas residents. Added to the $13,834 of real estate taxes, it gives $17,090.34 of itemized SALT and $283,059.84 of taxable income.", + "new": "Took the $16,100 standard deduction, which is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below it, but omitted the $196 non-itemizer charitable deduction; the applicable taxable income is $283,854.19.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It also took the standard deduction instead of the $17,090.34 SALT itemization (real estate taxes plus the Texas sales tax deduction) and left out the $222.64 of NIIT.", + "new": "Its standard deduction is appropriate under c_irs_sales_tax_2025, whose IRS 2025 table leaves SALT below $16,100; it omitted the $196 non-itemizer charitable deduction and, compared with the frozen reference, the $222.64 of NIIT.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "Used the standard deduction plus the $196 non-itemizer charitable deduction and missed the Texas general sales tax deduction that makes itemizing $17,090.34 the better choice. It also left out the $222.64 of NIIT that belongs in income tax before refundable credits.", + "new": "Used the standard deduction plus the $196 non-itemizer charitable deduction, as appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100. Its $68,112.00 is $0.09 below the exclusion record's $68,112.09 without NIIT; adjudication leaves NIIT inclusion ambiguous.", + "reason": "reference_outputs.csv.meta.json revisions[17] supports the standard deduction under c_irs_sales_tax_2025, while adjudication.json and reference_exclusions.json leave NIIT inclusion ambiguous and record $68,112.09 without it." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "That means it denied the $5,709 alimony deduction, took the standard deduction instead of $17,090.34 of itemized SALT including the Texas sales tax deduction, and left out the $222.64 of NIIT.", + "new": "That means it denied the $5,709 alimony deduction and omitted the $196 non-itemizer charitable deduction; its standard deduction is appropriate under c_irs_sales_tax_2025. Compared with the frozen reference, it also left out the $222.64 of NIIT.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "Took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction and missed the general sales tax deduction that brings Texas SALT itemization to $17,090.34. It also left out the $222.64 of NIIT.", + "new": "Took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, as appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100; its $68,111.95 is $0.14 below the exclusion record's $68,112.09 without NIIT.", + "reason": "prompt.md records this answer, reference_outputs.csv.meta.json revisions[17] establishes the standard deduction under c_irs_sales_tax_2025, and reference_exclusions.json records $68,112.09 without NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "That means it denied the $5,709 alimony deduction and took the standard deduction instead of $17,090.34 of itemized SALT (real estate taxes plus the Texas general sales tax deduction).", + "new": "That means it denied the $5,709 alimony deduction and omitted the $196 non-itemizer charitable deduction; its standard deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "It missed the Texas general sales tax deduction that makes itemized SALT $17,090.34, so taxable income came out at $289,759 instead of $283,059.84.", + "new": "Its standard deduction is appropriate under c_irs_sales_tax_2025, but it omitted the $196 non-itemizer charitable deduction; with alimony and charity, taxable income is $283,854.19 rather than its $289,759.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "Used the standard deduction plus the $196 non-itemizer charitable deduction ($283,854 of taxable income) instead of itemizing $17,090.34 of SALT, which includes the Texas general sales tax deduction. It also left out the $222.64 of NIIT.", + "new": "Used the standard deduction plus the $196 non-itemizer charitable deduction ($283,854 of taxable income), as appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100; its $68,111.95 is $0.14 below the exclusion record's $68,112.09 without NIIT.", + "reason": "prompt.md records this answer, reference_outputs.csv.meta.json revisions[17] establishes the standard deduction under c_irs_sales_tax_2025, and reference_exclusions.json records $68,112.09 without NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "The correct path uses $300,150.19 of AGI after alimony, $17,090.34 of itemized SALT including the Texas sales tax deduction, $67,834.07 of regular tax and $222.64 of NIIT.", + "new": "Under c_irs_sales_tax_2025, the path uses $300,150.19 of AGI after alimony, the $16,100 standard deduction plus $196 of non-itemizer charity, and $68,112.09 of regular tax, or $68,334.73 with NIIT; adjudication leaves NIIT inclusion ambiguous.", + "reason": "reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 at $68,334.73 with NIIT under c_irs_sales_tax_2025, while reference_exclusions.json records $68,112.09 without NIIT and keeps the frozen reference excluded." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "so taxable income came out at $148,321 instead of $283,059.84.", + "new": "so taxable income came out at $148,321 instead of $283,854.19 under c_irs_sales_tax_2025, which takes the standard deduction plus $196 of non-itemizer charity.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "Under 2026 law the regular tax on $283,059.84 of taxable income applies, with no AMT.", + "new": "Under 2026 law regular tax applies with no AMT, on $283,854.19 of taxable income under c_irs_sales_tax_2025, which takes the standard deduction plus $196 of non-itemizer charity.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "Took the $16,100 standard deduction (taxable income $284,050) and missed the general sales tax deduction that makes Texas itemized SALT $17,090.34.", + "new": "Took the $16,100 standard deduction, which is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below it, but omitted the $196 non-itemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "It also missed the Texas sales tax deduction that makes itemized SALT of $17,090.34 exceed the standard deduction.", + "new": "Its standard deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 sales tax table leaves SALT below $16,100.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "Treated itemized deductions as only the $13,834 of real estate taxes and missed the Texas general sales tax deduction that raises SALT to $17,090.34, above the standard deduction. Using the standard deduction plus the $196 non-itemizer charitable deduction gave $283,854 of taxable income instead of $283,059.84.", + "new": "Its standard deduction plus the $196 non-itemizer charitable deduction is the household's choice under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100; its $68,334.59 is $0.14 below the convention result of $68,334.73 with NIIT.", + "reason": "reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 at $68,334.73 with NIIT under c_irs_sales_tax_2025, while reference_exclusions.json records $68,112.09 without NIIT and keeps the frozen reference excluded." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "The correct result is $68,056.71: $283,059.84 of taxable income after alimony and $17,090.34 of itemized SALT, taxed through the 35% bracket, plus NIIT.", + "new": "Under c_irs_sales_tax_2025, taxable income is $283,854.19 after alimony, the $16,100 standard deduction and $196 of non-itemizer charity, giving $68,112.09 of regular tax or $68,334.73 with NIIT; $68,056.71 is the excluded frozen reference.", + "reason": "reference_outputs.csv.meta.json revisions[17] rechecks scenario_020 at $68,334.73 with NIIT under c_irs_sales_tax_2025, while reference_exclusions.json records $68,112.09 without NIIT and keeps the frozen reference excluded." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "It used the standard deduction plus the non-itemizer charitable deduction instead of itemizing $17,090.34 of SALT including the Texas sales tax deduction, and it left out NIIT.", + "new": "Its standard deduction plus the non-itemizer charitable deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100, and it left out NIIT compared with the frozen reference.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "and counted only $13,834 of property tax as SALT, missing the Texas general sales tax deduction that pushes itemized deductions to $17,090.34.", + "new": "and counted only $13,834 of property tax as SALT. Its standard deduction is appropriate under c_irs_sales_tax_2025 because the IRS 2025 table leaves SALT below $16,100, but it omitted the $196 non-itemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 in reference_outputs.csv.meta.json revisions[2] and scenario_020's revisions[17] recheck leaves SALT below the standard deduction, so the standard deduction plus $196 of non-itemizer charity gives taxable income of $283,854.19." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "SALT includes about $6,059 of estimated California income tax withheld on top of the $8,829 of real estate tax, for $14,888 under the $40,400 OBBBA cap. Charity is $15,393 less the 0.5%-of-AGI floor, so itemized deductions total $29,736. Adding the $6,000 senior deduction phased down by 6% of MAGI over $75,000 (to $3,951) and a $910 QBI deduction on the rental income gives taxable income of $74,552.59 and tax of $11,113.57. Most models went wrong in one of three ways: they used the joint QSS brackets and standard deduction, they left out or underestimated California income tax in SALT (or used the old $10,000 cap), or they applied the expired pre-TCJA rules (personal exemptions, 2%-floor miscellaneous deductions, 15%/25% brackets).", + "new": "The prompt lists no state income tax withheld or paid. The frozen reference's SALT adds the engine's $6,058.85 California withholding estimate on federal AGI to $8,829 of real estate tax, totaling $14,887.85 under the $40,400 cap. With charity after the 0.5%-of-AGI floor, itemized deductions are $29,735.52; the phased senior and $910 QBI deductions produce the frozen $74,552.59 taxable income and $11,113.57 tax. These totals depend on unlisted estimated withholding, and the output is excluded from scoring. The exclusion records $11,895.23 when SALT uses the engine's $2,505.87 California liability on policyengine-us 2.15.17, $12,013.38 with the corrected $1,968.81 liability, and $12,192.29 with no state income tax paid ($12,234.66 without the unlisted local sales tax estimate). Models used joint QSS treatment, the old SALT cap or pre-TCJA rules, or omitted deductions; their own California tax estimates or no state tax payment should not be faulted against the unlisted withholding estimate.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "It got single status, the phased $3,951 senior deduction and the $910 QBI deduction right, but put only the $8,829 of real estate tax in SALT. It left out the roughly $6,059 of California income tax withheld, which brings SALT to $14,888 under the $40,400 cap. That understated itemized deductions ($24,222 instead of $29,736) and inflated taxable income to $80,066.", + "new": "It used single status, the phased $3,951 senior deduction and the $910 QBI deduction, and included $8,829 of real estate tax in SALT. The prompt lists no state income tax withheld or paid; the frozen reference adds the engine's $6,058.85 withholding estimate, giving $29,736 of itemized deductions. The model's $24,222 of itemized deductions and $80,066 taxable income reflect a different treatment of that unlisted input and omission of the charitable floor.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It also left out the phased $3,951 senior deduction, and it dropped California income tax from SALT, which made itemizing look worse than it is.", + "new": "It also left out the phased $3,951 senior deduction. It put no state income tax in SALT, which the prompt does not list; the frozen reference adds the engine's $6,058.85 withholding estimate. As a single filer, the household itemizes under every recorded reading of that input.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "It left the roughly $6,059 of California income tax out of SALT, which put itemized deductions at $24,222 instead of $29,736.", + "new": "It itemized $24,222 without state income tax in SALT or the charitable floor. The prompt lists no state income tax withheld or paid; the frozen reference's $29,736 itemized total includes the engine's $6,058.85 withholding estimate.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "It also underestimated California income tax at $2,019 instead of about $6,059, and left out the senior and QBI deductions.", + "new": "It used its own $2,019 California tax estimate in SALT; the prompt lists no state income tax paid or withheld, and the frozen reference uses the engine's $6,058.85 withholding estimate. It also left out the senior and QBI deductions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "It also put California income tax at only $2,135 and left out the senior and QBI deductions.", + "new": "It used its own $2,135 California tax estimate in SALT; the prompt lists no state income tax paid or withheld, and the frozen reference uses the engine's $6,058.85 withholding estimate. It also left out the senior and QBI deductions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It itemized only real estate tax and charity ($24,222), leaving the roughly $6,059 of California income tax out of SALT. It also omitted both the phased $3,951 senior deduction and the $910 QBI deduction, so taxable income came out at $84,927 instead of $74,553.", + "new": "It itemized real estate tax and charity ($24,222), with no state income tax in SALT or charitable floor. The prompt lists no state income tax paid or withheld; the frozen reference adds the engine's $6,058.85 withholding estimate. It also omitted the phased $3,951 senior deduction and $910 QBI deduction, producing $84,927 taxable income compared with $74,553 under the frozen reference's assumptions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "It also never mentioned the phased $3,951 senior deduction, the $910 QBI deduction, or the California income tax that belongs in SALT. Its $11,335 implies taxable income about $1,000 above the correct $74,553.", + "new": "It also never mentioned the phased $3,951 senior deduction or the $910 QBI deduction. The prompt lists no state income tax paid or withheld; the frozen reference adds the engine's $6,058.85 withholding estimate to SALT. Its $11,335 implies taxable income about $1,000 above the frozen reference's $74,553 under that assumption.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "It also underestimated California income tax at $2,172 instead of about $6,059, and left out the senior and QBI deductions.", + "new": "It used its own $2,172 California tax estimate in SALT; the prompt lists no state income tax paid or withheld, and the frozen reference uses the engine's $6,058.85 withholding estimate. It also left out the senior and QBI deductions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "That fits itemizing only real estate tax and charity while skipping the California income tax in SALT, the phased $3,951 senior deduction and the $910 QBI deduction. The correct taxable income is $74,553.", + "new": "That is consistent with itemizing real estate tax and charity without state income tax in SALT, the phased $3,951 senior deduction or the $910 QBI deduction. The prompt lists no state tax payment or withholding; $74,553 is the frozen reference's taxable income under the engine's unlisted $6,058.85 withholding estimate.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "It capped SALT at the old $10,000, but the 2026 OBBBA cap of $40,400 allows the full $14,888 of real estate plus California income tax. It also left out the phased $3,951 senior deduction, so taxable income was $82,846 instead of $74,553.", + "new": "It capped SALT at the old $10,000 rather than the 2026 $40,400 cap. The frozen reference's $14,888 SALT includes the engine's $6,058.85 withholding estimate, although the prompt lists no state tax paid or withheld. It also left out the phased $3,951 senior deduction; its $82,846 taxable income compares with $74,553 under the frozen reference's assumptions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "about $8,400 below the correct $74,553. That means it overstated deductions beyond the $29,736 of itemized deductions, $3,951 senior deduction and $910 QBI deduction, or applied joint brackets the filer does not qualify for.", + "new": "about $8,400 below the frozen reference's $74,553. Relative to that reference's assumptions, this is consistent with higher deductions or joint brackets the filer does not qualify for. The reference's $29,736 itemized total includes the engine's $6,058.85 withholding estimate, an input the prompt does not list.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "Its $13,200 implies taxable income near $84,000 instead of $74,553.", + "new": "Its $13,200 implies taxable income near $84,000, compared with the frozen reference's $74,553 under the engine's unlisted $6,058.85 withholding estimate in SALT.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "It left California income tax out of SALT, so itemized deductions came to $24,222 instead of $29,736.", + "new": "It itemized $24,222 without state income tax in SALT or the charitable floor; the prompt lists no state income tax paid or withheld, and the frozen reference's $29,736 total adds the engine's $6,058.85 withholding estimate.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Total deductions are only $34,597, leaving $74,553 of taxable income and $11,113.57 of tax.", + "new": "Under the frozen reference's unlisted withholding assumption, deductions total $34,597, leaving $74,553 taxable income and $11,113.57 tax; the exclusion also records positive tax under the alternative readings.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "far below the correct $74,553.", + "new": "far below the frozen reference's $74,553, which depends on the engine's unlisted $6,058.85 withholding estimate in SALT.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "That is about $3,900 more in deductions than the correct $34,597, which fits an inflated SALT estimate or an unphased $6,000 senior deduction instead of the $3,951 phased amount.", + "new": "That implies about $3,900 more deductions than the frozen reference's $34,597, which includes the engine's unlisted $6,058.85 withholding estimate in SALT. The model gives no deduction amounts that locate the difference.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "Without a dependent child, single status applies: itemizing $29,736 (including California income tax in SALT) gives $74,553 of taxable income, taxed at 22% above $50,400.", + "new": "Without a dependent child, single status applies, with 22% above $50,400. The frozen reference itemizes $29,736 and yields $74,553 taxable income by adding the engine's $6,058.85 withholding estimate to SALT, although the prompt lists no state tax payment or withholding.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "which overstates total deductions by about $6,600 relative to the correct $34,597 (itemized $29,736 + senior $3,951 + QBI $910). The overstatement fits an inflated SALT or charitable figure or an unphased senior deduction.", + "new": "which implies about $6,600 more deductions than the frozen reference's $34,597 (itemized $29,736 + senior $3,951 + QBI $910). That reference includes the engine's unlisted $6,058.85 withholding estimate in SALT; the model reports a phased senior deduction but gives no itemized or QBI amounts to locate the difference.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "It estimated itemized deductions at $26,456, which understates California income tax in SALT (the reference uses about $6,059, for $14,888 of SALT). It also omitted the $910 QBI deduction on the rental income, leaving taxable income at $78,742 instead of $74,553.", + "new": "It estimated itemized deductions at $26,456. The prompt lists no state tax payment or withholding; the frozen reference instead uses the engine's $6,058.85 withholding estimate for $14,888 of SALT. It also omitted the $910 rental QBI deduction, leaving $78,742 taxable income compared with $74,553 under the frozen reference's assumptions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "With no dependent child the filer is single: itemizing $29,736 and applying single brackets (22% above $50,400) gives $11,113.57.", + "new": "With no dependent child the filer is single, with 22% above $50,400. The frozen reference's $29,736 itemized total and $11,113.57 tax depend on the engine's $6,058.85 withholding estimate in SALT, although the prompt lists no state tax paid or withheld.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "It handled filing status, senior deduction and QBI correctly, but estimated California income tax at only $1,974 instead of about $6,059, so SALT was $10,803 instead of $14,888. That understated itemized deductions and pushed taxable income to $78,092 instead of $74,553.", + "new": "It handled filing status, senior deduction and QBI correctly and used its own $1,974 California tax estimate, giving $10,803 SALT and $78,092 taxable income. The prompt lists no state tax paid or withheld; the frozen reference instead uses the engine's $6,058.85 withholding estimate, giving $14,888 SALT and $74,553 taxable income under that assumption.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "It estimated California income tax at only $1,867 (SALT $10,696 instead of $14,888) and set the QBI deduction to $0,", + "new": "It used its own $1,867 California tax estimate for $10,696 SALT; the prompt lists no state tax paid or withheld, and the frozen reference's $14,888 SALT uses the engine's $6,058.85 withholding estimate. It set the QBI deduction to $0,", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "It applied the 0.5% charitable floor correctly, but put only about $2,024 of California income tax in SALT ($10,853 instead of $14,888).", + "new": "It applied the 0.5% charitable floor and used its own approximately $2,024 California tax estimate for $10,853 SALT. The prompt lists no state tax paid or withheld; the frozen reference's $14,888 SALT instead uses the engine's $6,058.85 withholding estimate.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "It estimated California income tax at $2,700 instead of about $6,059, so SALT was $11,529 instead of $14,888. It also skipped the $910 QBI deduction on the rental income, leaving taxable income at $78,276 instead of $74,553.", + "new": "It used its own $2,700 California tax estimate for $11,529 SALT; the prompt lists no state tax paid or withheld, and the frozen reference's $14,888 SALT uses the engine's $6,058.85 withholding estimate. It also skipped the $910 rental QBI deduction, leaving $78,276 taxable income compared with $74,553 under the frozen reference's assumptions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "It itemized only real estate tax and charity ($24,222), leaving the roughly $6,059 of California income tax out of SALT. It also left out the $910 QBI deduction, so taxable income was $80,976 instead of $74,553.", + "new": "It itemized real estate tax and charity ($24,222), with no state income tax in SALT or charitable floor. The prompt lists no state income tax paid or withheld; the frozen reference adds the engine's $6,058.85 withholding estimate. It also left out the $910 QBI deduction, yielding $80,976 taxable income compared with $74,553 under the frozen reference's assumptions.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "The allowable deductions total only $34,597 against $109,149 of AGI, leaving $74,553 of taxable income.", + "new": "The frozen reference deducts $34,597 from $109,149 AGI, leaving $74,553 taxable income under the engine's unlisted $6,058.85 withholding estimate in SALT; the exclusion records positive tax under the alternative readings too.", + "reason": "adjudication.json and reference_exclusions.json exclude this federal output because $6,058.85 is an unlisted engine withholding estimate, as prior_amendments.json previously explained." + }, + { + "case_id": "us__scenario_023__snap", + "field": "case_annotation", + "old": "For SNAP, PolicyEngine uses the USDA definition of disabled (7 CFR 271.2), which requires receiving a disability-based benefit such as SSI or SSDI.", + "new": "For SNAP, the USDA definition of disabled (7 CFR 271.2) requires receiving a qualifying disability-based benefit such as SSI or SSDI.", + "reason": "prior_amendments.json's engine-definition correction, reference_exclusions.json's scenario_023 SNAP note, and reference_outputs.csv.meta.json revisions[17] show that the frozen engine granted SNAP disability status from SSI criteria alone and the later engine omits the disability-based Medi-Cal route." + }, + { + "case_id": "us__scenario_023__snap", + "field": "case_annotation", + "old": "The benefit is $298 − $262.80 = $35.20 a month for January–September and $48.18 a month for October–December under FY2027 parameters, for $461.34 a year.", + "new": "The frozen reference gives $298 − $262.80 = $35.20 a month for January–September and $48.18 for October–December under the engine's projected FY2027 parameters, totaling $461.34. The release's c_snap_hold_fy2026 convention holds FY2026 for all 12 months; on this capped reading, the current engine with SNAP rounding fixes gives $34/month ($408/year).", + "reason": "reference_outputs.csv.meta.json revisions[0] (c_snap_hold_fy2026), [12] and [14] (SNAP rounding fixes), and [17] and [19] (scenario_023 SNAP rechecks) give $34/month ($408/year) on the capped reading, while reference_exclusions.json retains $461.34 as an excluded frozen value." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "which gives about $35–48/month.", + "new": "which gives about $35–48/month in the frozen reference's projected schedule; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "claude-opus-5", + "old": "Monthly gross of $2,120 is under California's 200% FPL BBCE limit, and capped net income of $876.51 is under the 100% FPL net limit of $1,304. Since $298 minus 30% of net is still $35.20/month, the benefit is positive, not zero.", + "new": "Monthly gross of $2,120 passes California's 200% FPL BBCE test, and capped net income of $876.51 passes the net-income test, whose monthly limit is rounded up to a whole dollar. The frozen reference gives $35.20/month for January–September; under c_snap_hold_fy2026 and the SNAP rounding fixes, this capped reading gives $34/month ($408/year), so the benefit remains positive.", + "reason": "reference_outputs.csv.meta.json revisions[15] requires rounding monthly income standards up, and revisions[17] and [19] recheck this capped reading at $34/month ($408/year) under c_snap_hold_fy2026 and the SNAP rounding fixes, while reference_exclusions.json retains the frozen value only as excluded." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "which yields $35–48/month rather than the maximum.", + "new": "which yields $35–48/month in the frozen reference's projected schedule; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "and only about $35–48/month in benefits.", + "new": "and about $35–48/month in benefits in the frozen reference's projected schedule; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "and the benefit is $35.20/month, not the maximum.", + "new": "and the frozen reference gives $35.20/month for January–September; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "reference_outputs.csv.meta.json revisions[0] (c_snap_hold_fy2026), [12] and [14] (SNAP rounding fixes), and [17] and [19] (scenario_023 SNAP rechecks) give $34/month ($408/year) on the capped reading, while reference_exclusions.json retains $461.34 as an excluded frozen value." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "and the allotment is $35.20/month for most of the year.", + "new": "and the frozen reference's allotment is $35.20/month for January–September, followed by a projected FY2027 increase; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "glm-5.3", + "old": "and a benefit of $35.20/month.", + "new": "and the frozen reference's $35.20/month for January–September; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "reference_outputs.csv.meta.json revisions[0] (c_snap_hold_fy2026), [12] and [14] (SNAP rounding fixes), and [17] and [19] (scenario_023 SNAP rechecks) give $34/month ($408/year) on the capped reading, while reference_exclusions.json retains $461.34 as an excluded frozen value." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "and the benefit is $35.20–$48.18/month.", + "new": "and the frozen reference gives $35.20–$48.18/month using projected FY2027 parameters for October–December; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "and 30% of that ($262.80) is below the $298 maximum, so the benefit is positive.", + "new": "and the rounded expected contribution is $264, below the $298 maximum, giving $34/month ($408/year) on this capped reading under c_snap_hold_fy2026 and the SNAP rounding fixes.", + "reason": "reference_outputs.csv.meta.json revisions[0] (c_snap_hold_fy2026), [12] and [14] (SNAP rounding fixes), and [17] and [19] (scenario_023 SNAP rechecks) give $34/month ($408/year) on the capped reading, while reference_exclusions.json retains $461.34 as an excluded frozen value." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-6-sol", + "old": "and the benefit is $35.20/month for most months.", + "new": "and the frozen reference gives $35.20/month for January–September, followed by a projected FY2027 increase; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "grok-4.6", + "old": "and the benefit is $35.20/month.", + "new": "and the frozen reference gives $35.20/month for January–September; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "reference_outputs.csv.meta.json revisions[0] (c_snap_hold_fy2026), [12] and [14] (SNAP rounding fixes), and [17] and [19] (scenario_023 SNAP rechecks) give $34/month ($408/year) on the capped reading, while reference_exclusions.json retains $461.34 as an excluded frozen value." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "kimi-k2.6", + "old": "and the benefit is $35.20–$48.18/month.", + "new": "and the frozen reference gives $35.20–$48.18/month using projected FY2027 parameters for October–December; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "muse-wonder", + "old": "and the benefit is $35.20/month for most months.", + "new": "and the frozen reference gives $35.20/month for January–September, followed by a projected FY2027 increase; c_snap_hold_fy2026 with the SNAP rounding fixes gives $34/month ($408/year) on this capped reading.", + "reason": "prior_amendments.json identifies the higher October–December amount as the frozen engine's projection, while reference_outputs.csv.meta.json revisions[0] holds FY2026 all year and revisions[17] and [19] give $34/month ($408/year) on this capped reading." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "ox-alpha", + "old": "The model correctly capped the shelter deduction at $744, but it used a $204 standard deduction instead of $209 and applied FY2026 parameters to all 12 months. From October 2026, the FY2027 maximum allotment rises to $304.68 and the expected contribution falls to $256.50, giving $48.18/month for October–December. That makes the annual total $461.34 rather than $402.", + "new": "The model correctly capped the shelter deduction at $744 and applied FY2026 parameters to all 12 months, as c_snap_hold_fy2026 requires, but used a $204 standard deduction instead of $209. The frozen reference reaches $461.34 using the engine's projected FY2027 $304.68 maximum and $256.50 contribution ($48.18/month) for October–December; on the capped reading, the current engine with the convention and SNAP rounding fixes gives $408 ($34 × 12), compared with the model's $402.", + "reason": "prior_amendments.json's ox-alpha correction and reference_outputs.csv.meta.json revisions[0], [17] and [19] establish that applying FY2026 all year follows c_snap_hold_fy2026 and gives $408 with rounding fixes, whereas $461.34 is the excluded frozen projection." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "The correct calculation gives net income of $876.51 and a benefit of $35.20/month.", + "new": "The frozen reference gives net income of $876.51 and $35.20/month for January–September; under c_snap_hold_fy2026 and the SNAP rounding fixes, this capped reading gives $34/month ($408/year).", + "reason": "reference_outputs.csv.meta.json revisions[0] (c_snap_hold_fy2026), [12] and [14] (SNAP rounding fixes), and [17] and [19] (scenario_023 SNAP rechecks) give $34/month ($408/year) on the capped reading, while reference_exclusions.json retains $461.34 as an excluded frozen value." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "assumed there was no modeled Minnesota employee payroll tax", + "new": "said there was no modeled mandatory Minnesota employee payroll tax", + "reason": "prompt.md retains the model's mandatory-tax qualifier, consistent with the employer-choice ambiguity recorded in adjudication.json." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "but it said Minnesota levies no employee payroll tax", + "new": "and it said Minnesota has no mandatory employee payroll tax", + "reason": "prompt.md says the model ruled out mandatory Minnesota employee payroll tax, while reference_exclusions.json records that the Paid Leave employee deduction is optional." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "It said Minnesota has no paid family leave employee contribution in 2026, which is wrong: Minnesota Paid Leave premiums began January 1, 2026. PolicyEngine includes the 0.44% employee share (0.44% × $29,000 = $127.60), which raises the total to $2,346.10.", + "new": "It reported $2,218.50 and said no state disability or paid family leave employee contribution applied in this scenario. Minnesota Paid Leave premiums began January 1, 2026; the frozen reference assumes a $127.60 employee deduction, but the employer may pay that share itself and the prompt does not specify its choice.", + "reason": "prompt.md qualifies the model's statement as applying to this scenario, and reference_exclusions.json and prior_amendments.json establish that the employee deduction is optional." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It assumed no Minnesota employee payroll tax is modeled. PolicyEngine does model the Minnesota Paid Leave employee contribution, at 0.44% × $29,000 = $127.60, on top of $2,218.50 in FICA.", + "new": "It assumed no mandatory Minnesota employee state payroll tax was modeled and answered $2,218.50. The frozen reference assumes a $127.60 Minnesota Paid Leave deduction (0.44% of wages); the employer may pay it instead, and the unspecified choice also permits federal tax alone, $2,218.50.", + "reason": "prompt.md retains the model's mandatory-tax qualifier, and reference_exclusions.json and prior_amendments.json establish that the modeled Paid Leave share is an optional employer deduction." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "It correctly included the Minnesota Paid Leave employee share but used 0.35%, which is half of the original 0.7% premium. The 2026 premium is 0.88%, so the employee share is 0.44%, or $127.60 rather than $101.50. That understated the total by $26.10.", + "new": "It added a Minnesota Paid Leave share at 0.35% ($101.50), giving $2,320. The frozen reference assumes the maximum deduction, half of the 0.88% premium or 0.44% of wages ($127.60), so its answer is $26.10 below that reference; the employer may deduct less or pay the share itself, and the prompt does not specify its choice.", + "reason": "reference_exclusions.json permits an employee share from zero to 0.44%, and prior_amendments.json corrected presenting the maximum deduction as the uniquely correct contribution." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "The 2026 employee share is 0.44% of the 0.88% premium, which is $127.60, for a total of $2,346.10.", + "new": "The frozen reference adds the maximum deductible employee share, half of the 0.88% premium or 0.44% of wages ($127.60), for $2,346.10; the employer may pay that share itself, and the prompt does not specify its choice.", + "reason": "prompt.md derives $127.60 from 0.44% of wages, not 0.44% of the premium, and reference_exclusions.json records that this maximum deduction depends on the employer's unspecified choice." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "glm-5.2", + "old": "The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave contribution = $2,346.10.", + "new": "The frozen reference is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave = $2,346.10, assuming the maximum employer deduction; the employer may pay that share itself, giving $2,218.50, and the prompt does not specify its choice.", + "reason": "reference_exclusions.json and adjudication.json retain $2,346.10 as an excluded reference with a $2,218.50 alternative, repeating the uniquely-correct-total error corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "The correct total is $2,218.50 in FICA plus the $127.60 Minnesota Paid Leave share.", + "new": "The frozen reference assumes a $127.60 Minnesota Paid Leave deduction (0.44% of wages); the employer may pay it instead, and the unspecified choice also permits federal tax alone, $2,218.50.", + "reason": "reference_exclusions.json and adjudication.json retain $2,346.10 as an excluded reference with a $2,218.50 alternative, consistent with the employer-choice qualification restored in prior_amendments.json." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It treated the absence of any stated employee-side tax as meaning none exists. Minnesota Paid Leave, effective 2026, imposes a 0.44% employee share ($127.60 on $29,000), which PolicyEngine withholds.", + "new": "It reported federal tax alone, $2,218.50, saying no other employee-side payroll tax was indicated. The frozen reference assumes a $127.60 Minnesota Paid Leave deduction (0.44% of wages); the employer may pay it instead, and the unspecified choice also permits federal tax alone, $2,218.50.", + "reason": "prompt.md says no other employee-side tax was indicated, and reference_exclusions.json and prior_amendments.json establish that Minnesota permits rather than imposes the maximum employee deduction." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "annotation", + "model": "kimi-k2.6", + "old": "The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave contribution = $2,346.10.", + "new": "The frozen reference is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave = $2,346.10, assuming the maximum employer deduction; the employer may pay that share itself, giving $2,218.50, and the prompt does not specify its choice.", + "reason": "reference_exclusions.json and adjudication.json retain $2,346.10 as an excluded reference with a $2,218.50 alternative, repeating the uniquely-correct-total error corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "case_annotation", + "old": "Every model got employee FICA right or nearly right ($1,798 Social Security + $420.50 Medicare = $2,218.50). The $127.60 gap is the Minnesota Paid Leave premium, which started January 1, 2026.", + "new": "Most listed models reported employee FICA alone ($1,798 Social Security + $420.50 Medicare = $2,218.50), sometimes rounded to $2,219; others included a Paid Leave share, submitted inconsistent amounts or returned no value. The $127.60 difference between unrounded FICA and the frozen reference is the maximum Minnesota Paid Leave deduction; premiums began January 1, 2026.", + "reason": "prompt.md records missing outputs and inconsistent submitted amounts, and prior_amendments.json distinguishes those failures from the optional Paid Leave gap recorded in reference_exclusions.json." + }, + { + "case_id": "us__scenario_032__payroll_tax", + "field": "case_annotation", + "old": "The prompt never says which choice this employer makes, so most models took the second reading.", + "new": "The prompt never says which choice this employer makes; most listed models reported federal FICA alone, and only grok-4.7 explicitly cited optional paid-leave withholding.", + "reason": "prompt.md and the earlier case-note correction in prior_amendments.json show that only grok-4.7 explicitly invoked optional withholding, so federal-only answers do not establish that most models chose that interpretation." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "The model used the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835, which gave $830.40. It then submitted $1,020, a number its own arithmetic does not support. The correct amount is 3% × $27,345 = $820.35.", + "new": "The model used 2025's $12,500 deduction to calculate $830.40, then submitted $1,020 without supporting arithmetic. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "The model used Louisiana's 2025 single standard deduction of $12,500 without the 2026 inflation indexing that raises it to $12,835. That overstated taxable income by $335 and tax by $10.05, giving $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "Louisiana taxes income at a flat 3% after a $12,835 single standard deduction, which gives $820.35 on $40,180 of AGI.", + "new": "Louisiana has a flat 3% individual income tax. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "Louisiana's only offset is a $12,835 standard deduction, and the personal exemption was eliminated. That leaves $27,345 taxable at 3%, or $820.35.", + "new": "The reform eliminated the personal exemption, and the standard deduction does not wipe out this household's liability. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "The model applied the flat 3% rate correctly but used the 2025 standard deduction of $12,500 instead of the inflation-indexed 2026 amount of $12,835. That produced $830.40 instead of $820.35.", + "new": "The model applied the flat 3% rate to income less 2025's $12,500 deduction and obtained $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "The correct amount is 3% × ($40,180 − $12,835) = $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "The model used the $12,500 deduction instead of the 2026 indexed $12,835. It then made an unsupported downward adjustment for 'AGI rounding' to reach $812, when Louisiana AGI is exactly $40,180 and tax is $820.35.", + "new": "The model used 2025's $12,500 deduction to calculate about $830, then made an unsupported downward adjustment for 'AGI rounding' to reach $812; Louisiana AGI is $40,180. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "The model used the $12,500 single standard deduction without the 2026 inflation adjustment to $12,835. That put taxable income at $27,680 instead of $27,345 and tax at $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It also used the unindexed $12,500 deduction instead of $12,835. Louisiana taxable income is $40,180 − $12,835 = $27,345, and tax is $820.35, not $747.", + "new": "It also used 2025's $12,500 deduction. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The 2026 deduction of $12,835 gives $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "The model assumed a $12,500 standard deduction for 2026, but Louisiana indexes it to inflation starting in 2026, which makes it $12,835. That inflated the tax by $10.05, to $830.40.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "The reform replaced both with a single standard deduction, $12,835 in 2026. That gives $27,345 taxable and $820.35 tax, not $935.", + "new": "The reform replaced those offsets with a single standard deduction. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "The model used a 4.25% rate instead of the 3% flat rate and allowed only a $4,500 exemption instead of the $12,835 standard deduction. It also wrongly cut AGI to $34,791 by subtracting the $5,389 ESI premium. Louisiana AGI is $40,180, and tax is $820.35.", + "new": "The model used a 4.25% rate and a $4,500 exemption instead of the reformed 3% rate and standard deduction, and subtracted the $5,389 ESI premium to cut AGI to $34,791 rather than $40,180. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "The model used the unindexed $12,500 standard deduction instead of the 2026 amount of $12,835. That produced $830.40 instead of 3% × $27,345 = $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "The model applied the 3% flat rate after the 2025-level $12,500 deduction and did not apply the 2026 inflation indexing to $12,835. That overstated tax at $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "Current law taxes $40,180 − $12,835 at a flat 3%, which gives $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "The correct calculation is one 3% rate on $40,180 minus the $12,835 2026 standard deduction, which gives $820.35, not $724.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The correct calculation is 3% × ($40,180 − $12,835) = $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "Under 2026 law, Louisiana taxes $40,180 minus the $12,835 standard deduction at a flat 3%, which gives $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "The correct calculation is 3% of ($40,180 − $12,835 standard deduction) = $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "Under the 2026 rules, the $12,835 standard deduction and the flat 3% rate give $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "Current law gives 3% × ($40,180 − $12,835) = $820.35, not $1,042.55.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "The model used a $12,500 single standard deduction for 2026, but Act 11 of 2024 indexes it to inflation starting in 2026, which makes it $12,835. That overstated tax at $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "The model applied the 2025 base deduction of $12,500 instead of the inflation-indexed 2026 deduction of $12,835. That produced $27,680 instead of $27,345 of taxable income and $830.40 of tax instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "The model claimed deductions fully offset the tax, but $40,180 of AGI far exceeds the $12,835 Louisiana standard deduction. That leaves $27,345 taxed at 3%, or $820.35.", + "new": "The model claimed deductions fully offset the tax, but they leave positive Louisiana liability on $40,180 of AGI. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "The correct path is the flat 3% rate on $40,180 minus the $12,835 2026 standard deduction, which yields $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "The model used the $12,500 standard deduction without the 2026 inflation adjustment to $12,835. That produced $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "Louisiana's 2026 deduction is $12,835, so taxable income is $27,345 and tax is $820.35, not $830.40.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "The model used the 2025 standard deduction of $12,500 instead of the inflation-indexed 2026 amount of $12,835. That overstated tax by $10.05, at $830.40.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The model subtracted the unindexed $12,500 deduction instead of the 2026 deduction of $12,835. That gave $830.40 instead of 3% × $27,345 = $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "The model used $12,500 as the 2026 single standard deduction and missed the inflation indexing that raises it to $12,835. That produced $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The 2026 indexed deduction is $12,835, which lowers taxable income to $27,345 and tax to $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "The model applied the $12,500 single deduction without the 2026 inflation adjustment to $12,835. That gave $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "The 2026 Louisiana single deduction is $12,835, so the correct tax is 3% × $27,345 = $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "That figure understates the flat 3% tax on $40,180 minus the $12,835 2026 standard deduction, which is $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "The model subtracted the federal standard deduction instead of Louisiana's own $12,835 deduction. It also applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate. The correct result is $820.35.", + "new": "The model subtracted the federal standard deduction rather than Louisiana's own deduction and applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "The 2026 deduction of $12,835 gives $27,345 taxable and $820.35 tax.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "The model applied the 2025 base deduction of $12,500 instead of the inflation-indexed 2026 deduction of $12,835. That overstated tax at $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-build-0.1", + "old": "The correct calculation is 3% × ($40,180 − $12,835) = $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "Louisiana's 2026 standard deduction is $12,835, which yields $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k2.6", + "old": "The correct value is 3% × ($40,180 − $12,835) = $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "The model subtracted the 2025-level $12,500 deduction instead of the 2026 inflation-indexed $12,835. That gave $830.40 instead of $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "The $12,835 2026 standard deduction against $40,180 at 3% gives $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "The model used the $12,500 single deduction without the 2026 inflation indexing to $12,835. That overstated taxable income by $335 and tax at $830.40.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "The model applied the unindexed $12,500 standard deduction instead of the 2026 amount of $12,835. That produced $830.40 instead of 3% × $27,345 = $820.35.", + "new": "The model used 2025's $12,500 deduction and calculated $830.40. The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "Louisiana taxes $40,180 minus its own $12,835 deduction at a flat 3%, which gives $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "That figure implies a higher rate or a much smaller deduction than Louisiana's 2026 rules, which are a flat 3% on $40,180 minus the $12,835 standard deduction, or $820.35.", + "new": "The retained engine reference is $820.35 using its computed $12,835 deduction; RIB 26-019 published $12,838 after the freeze, giving $820.26, and this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the excluded $820.35 engine calculation from the published $12,838 deduction and $820.26 tax." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Louisiana's 2025 reform set a flat 3% rate and a $12,500 single standard deduction, and that deduction is inflation-indexed starting in 2026, which makes it $12,835. The correct calculation is 3% × ($40,180 − $12,835) = $820.35.", + "new": "Louisiana's 2025 reform set a flat 3% rate and a $12,500 single standard deduction, indexed from 2026 under R.S. 47:294(B). PolicyEngine computed $12,835 from published CPI-U, giving the retained reference of 3% × ($40,180 − $12,835) = $820.35. Louisiana's first published 2026 return deduction was $12,838 in RIB 26-019 on 2026-09-28, giving $820.26; this output is excluded under d994.", + "reason": "reference_exclusions.json (scenario_051, d994) and adjudication.json distinguish the retained, excluded engine calculation from Louisiana's published amount, correcting the unsourced $12,835 claim previously addressed in prior_amendments.json." + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Most models used the unindexed $12,500 figure and got $830.40. The rest used repealed pre-2025 law, including the $4,500 exemption, the 1.85%/3.5% brackets and the deduction for federal income tax, or they wrongly subtracted the $5,389 ESI premium from AGI.", + "new": "The largest final-answer group, 22 of the 48 models, used $12,500 and answered $830.40 or $830; four more computed about $830 using that deduction and then submitted another amount. Fourteen used repealed rules, including the $4,500 exemption or deduction, the 1.85%/3.5% brackets, or the deduction for federal income tax; six of these also subtracted the $5,389 ESI premium from AGI. The remaining eight claimed no tax or an offsetting deduction, gave no calculation, or gave no answer.", + "reason": "prompt.md shows 22 final $830.40/$830 answers, four different submissions after an approximately $830 calculation, 14 uses of repealed rules and eight other or missing answers, repeating the overgeneralization identified in prior_amendments.json." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It disallowed the $43 traditional IRA deduction under the active-participant phase-out, which left AGI at $504,822 and taxable income at $472,622. The reference deducts $43.28 of IRA contributions, giving taxable income of $472,578.56, and that $43.44 at the 32% rate accounts for the whole $13.89 gap.", + "new": "It correctly disallowed the $43 traditional IRA deduction under the active-participant phase-out, leaving rounded AGI at $504,822 and taxable income at $472,622. The frozen reference incorrectly deducts $43.28, giving taxable income of $472,578.56; the $43.44 difference also includes $0.16 from rounding the 401(k) deferral. Its $104,225.30 is $0.04 above the exclusion's recorded corrected value, $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) disallows this active participant's IRA deduction and records corrected tax $104,225.26, while prompt.md gives this model $104,225.30 using a rounded $926 401(k) deferral, repeating the issue corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "Taxable income of $472,578.56 under the 2026 MFJ schedule gives $103,865.78 of ordinary tax, plus $127.20 on the qualified dividends at 15% and $218.42 of NIIT, so the round guess is about $789 too high.", + "new": "The frozen reference uses taxable income of $472,578.56 and ordinary tax of $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT. It incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26, so the round guess is about $775 above that value.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "It never subtracted the $926 traditional 401(k) deferral or the $43 IRA contribution, so AGI came out at $505,748 instead of $504,778.56.", + "new": "Its reported AGI of $505,748 fails to reflect the $926 traditional 401(k) deferral in its arithmetic. Disallowing the $43 IRA deduction agrees with the exclusion's active-participant phase-out correction, which gives AGI of $504,821.84; $504,778.56 is the frozen reference's AGI, including the defective IRA deduction.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) disallows this active participant's IRA deduction, while prompt.md shows that the model explicitly subtracted the 401(k) deferral in its formula but failed to reflect it in the reported AGI." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The contributions are annual totals, so only $926.16 and $43.28 reduce income, giving taxable income of $472,578.56.", + "new": "The contributions are annual totals: the $926.16 401(k) deferral reduces income, while the $43.28 IRA deduction is fully phased out under the exclusion's correction. Taxable income of $472,578.56 belongs to the frozen reference, which incorrectly takes the IRA deduction; the recorded corrected tax is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "The correct steps are AGI of $504,778.56, taxable income of $472,578.56, and $103,865.78 of ordinary tax, plus $127.20 QD tax and $218.42 NIIT.", + "new": "The frozen reference's steps are AGI of $504,778.56, taxable income of $472,578.56, and $103,865.78 of ordinary tax, plus $127.20 QD tax and $218.42 NIIT. That calculation incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "which lowered AGI to $488,614 instead of $504,778.56.", + "new": "which lowered AGI to $488,614 instead of $504,821.84 under the exclusion's IRA phase-out correction; $504,778.56 is the frozen reference's AGI, including the defective $43.28 IRA deduction.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "Taxable income of $472,578.56 gives $103,865.78 of ordinary tax, plus $127.20 at 15% on the qualified dividends and $218.42 NIIT, so its answer is about $9,200 too high.", + "new": "The frozen reference uses taxable income of $472,578.56 and ordinary tax of $103,865.78, plus $127.20 at 15% on qualified dividends and $218.42 NIIT. That calculation incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26, still about $9,200 below its answer.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "which lowered AGI to $488,614 instead of $504,778.56. It then applied higher pre-OBBBA rates, so its regular tax came to about $117,547. The 2026 rules give $103,865.78 of ordinary tax on $472,578.56 of taxable income, plus $127.20 QD tax.", + "new": "which lowered AGI to $488,614 instead of $504,821.84 under the exclusion's IRA phase-out correction. It then applied higher pre-OBBBA rates, so its regular tax came to about $117,547. The frozen reference's AGI of $504,778.56 and taxable income of $472,578.56 include the defective IRA deduction; its ordinary tax is $103,865.78 plus $127.20 QD tax. The exclusion's recorded corrected total is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "Wages are reduced only by the $926.16 401(k) deferral, so taxable income is $472,578.56.", + "new": "Wages are reduced by the $926.16 401(k) deferral. The frozen reference's taxable income of $472,578.56 also includes an incorrect $43.28 IRA deduction; the exclusion's IRA phase-out correction disallows it and records tax of $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "It overstated AGI at about $505,800; after the $926.16 401(k), the $43.28 IRA deduction, and the $3,000 capital-loss limit, AGI is $504,778.56. It used a $32,400 standard deduction instead of $32,200 and rounded ordinary tax up to about $105,200 instead of $103,865.78.", + "new": "It overstated AGI at about $505,800; after the $926.16 401(k) deferral and $3,000 capital-loss deduction, AGI is $504,821.84 under the exclusion's IRA phase-out correction. The frozen reference's AGI of $504,778.56 incorrectly includes the $43.28 IRA deduction. It used a $32,400 standard deduction instead of $32,200 and rounded ordinary tax up to about $105,200, compared with the frozen reference's $103,865.78; the recorded corrected total tax is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Taxable income of $472,578.56 reaches the 32% bracket: $82,048 of tax through $403,550, plus 32% of the remaining $68,180.56, totals $103,865.78. Adding $127.20 of QD tax and $218.42 of NIIT gives $104,211.41.", + "new": "The frozen reference's $472,578.56 of taxable income includes $848 of qualified dividends, leaving $471,730.56 of ordinary income. Its ordinary tax is $82,048 through $403,550 plus 32% of $68,180.56, or $103,865.78; its displayed QD tax and NIIT are $127.20 and $218.42, and its stored total rounds to $104,211.41. That reference incorrectly takes the $43.28 IRA deduction; the exclusion's recorded corrected value is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records the frozen IRA deduction as defective and corrected tax as $104,225.26, while prompt.md identifies the $848 of qualified dividends that must be separated from total taxable income to obtain the stated ordinary-income remainder." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "Against about $508,748 of gross income, only the $926.16 401(k) deferral, the $43.28 IRA deduction, the $3,000 capital loss, and the $32,200 standard deduction apply. That leaves $472,578.56 of taxable income and $104,211.41 of tax.", + "new": "Against about $508,748 of gross income, the $926.16 401(k) deferral, $3,000 capital-loss deduction, and $32,200 standard deduction apply; the $43.28 IRA deduction is fully phased out under the exclusion's correction. The frozen reference incorrectly takes that IRA deduction, giving taxable income of $472,578.56 and tax of $104,211.41; the recorded corrected tax is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "It reported only regular tax and left out the 3.8% NIIT on $5,748 of net investment income ($218.42), which counts toward federal income tax before refundable credits. It also disallowed the $43 IRA deduction (taxable income of $472,622 instead of $472,578.56), which adds $13.89.", + "new": "It reported only regular tax and left out the $218.42 NIIT included in the frozen reference; the exclusion separately flags whether this output includes NIIT as an unlisted input. It correctly disallowed the $43 IRA deduction, giving rounded taxable income of $472,622; the frozen reference's $472,578.56 incorrectly includes that deduction. The exclusion's recorded IRA-corrected value is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records the IRA deduction as an engine defect and NIIT scope as an unlisted input, and the sidecar's 2026-10-13 recheck retains the exclusion despite the engine reaching $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It also disallowed the $43 IRA deduction, which adds $13.89 at 32%.", + "new": "It correctly disallowed the $43 IRA deduction under the exclusion's active-participant phase-out correction; the frozen reference incorrectly takes that deduction.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It deducted only the 401(k) deferral and the capital loss and left out the $43.28 traditional IRA deduction, so taxable income was $472,622. The reference deducts the IRA contribution, giving taxable income of $472,578.56, and that $43.44 at 32% accounts for the whole $13.89 gap.", + "new": "It deducted the 401(k) deferral and capital loss and omitted the IRA deduction, which agrees with the exclusion's active-participant phase-out correction. Its rounded taxable income of $472,622 is $43.44 above the frozen reference's $472,578.56; that difference includes the defective $43.28 IRA deduction and $0.16 of input rounding. Its $104,225.30 is $0.04 above the exclusion's recorded corrected value, $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) disallows this active participant's IRA deduction and records corrected tax $104,225.26, while prompt.md gives this model $104,225.30 using rounded inputs, repeating the issue corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "Its AGI of $504,779 is right, but it then applied TCJA expiration: a $16,950 standard deduction, $5,184 of personal exemptions, and pre-TCJA rates. OBBBA kept the $32,200 MFJ standard deduction and 24%/32% brackets for 2026, giving $472,578.56 of taxable income and $103,865.78 of ordinary tax.", + "new": "Its AGI of $504,779 matches the frozen reference, which incorrectly takes the $43.28 IRA deduction; the exclusion's IRA phase-out correction gives AGI of $504,821.84. It then applied TCJA expiration: a $16,950 standard deduction, $5,184 of personal exemptions, and pre-TCJA rates. OBBBA kept the $32,200 MFJ standard deduction and 24%/32% brackets for 2026. The frozen reference's taxable income of $472,578.56 and ordinary tax of $103,865.78 reflect the defective IRA deduction; the recorded corrected total tax is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "It also disallowed the $43.28 IRA deduction, which raises taxable income by $43.44 at 32%.", + "new": "It correctly disallowed the $43.28 IRA deduction under the exclusion's active-participant phase-out correction. The $43.44 difference from the frozen reference's taxable income also includes input rounding; the exclusion's recorded corrected tax is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) and adjudication.json identify the $43.28 IRA deduction as an engine defect under 26 U.S.C. 219(g), retaining frozen tax $104,211.41 and recording corrected tax $104,225.26." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Most wrong answers came from four mistakes: assuming the TCJA sunset (personal exemptions and the old 28%/33% schedule), subtracting the ESI premium from wages, adding the Additional Medicare Tax to income tax, or leaving out or misstating the NIIT.", + "new": "Common gaps to the frozen reference involved assuming the TCJA sunset (personal exemptions and the old 28%/33% schedule), subtracting the ESI premium from wages, adding the Additional Medicare Tax to income tax, or differing from the reference's NIIT treatment; the exclusion record leaves whether this output includes NIIT unresolved.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) identifies r25_niit_in_federal_output as an unlisted input, and reference_outputs.csv.meta.json's 2026-10-13 recheck retains the exclusion because the NIIT scope issue remains unresolved despite the engine reaching the IRA-corrected value." + }, + { + "case_id": "us__scenario_056__snap", + "field": "case_annotation", + "old": "The household is SNAP-eligible. Gross income of $1,486.54/mo is 112% of FPL, and net income of $805.03 is 61% of FPL. PolicyEngine counts only the $11,949 of real estate taxes ($995.75/mo) as shelter costs. The listed home mortgage interest is a tax itemized-deduction input and does not enter SNAP shelter costs. That gives an uncapped $395.49 excess shelter deduction and a formula benefit of $304.68 − $241.50 = $63.18. New Jersey's state minimum SNAP benefit of $95/mo raises this to $1,140/yr. No model applied the NJ $95 minimum. The $0 answers either stopped at a non-positive formula benefit or wrongly failed the income or asset tests. The ~$1,900 answers added mortgage interest to shelter costs and hit the shelter cap.", + "new": "The $1,140 annual amount is a frozen reference excluded from scoring because work hours and occupied-home mortgage status are unlisted. Its historical derivation assumed 40 weekly work hours and property-tax-only shelter costs, producing a $63.18 formula benefit raised to New Jersey's $95 monthly minimum. Mortgage payments including interest count as SNAP shelter costs if the household occupies the mortgaged home. Some $0 answers used the zero-hours ABAWD reading recognized by the developer decision. The sidecar's 2026-09-29 recheck reproduces $1,908/yr with 40 weekly hours and mortgage interest counted, and $95/yr under the engine's zero-hours and county defaults; later rechecks retain $95 while preserving the frozen $1,140 and exclusion.", + "reason": "prior_amendments.json and reference_exclusions.json recognize the occupied-home mortgage and zero-hours readings, adjudication.json excludes the output, and reference_outputs.csv.meta.json's excluded_outputs_rechecked retains $1,140 while reporting $95 and a $1,908 alternative." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "Only the $11,949 of real estate taxes counts, so the excess shelter deduction is $395.49/mo and net income is $805/mo. The resulting $63/mo formula benefit is raised to New Jersey's $95 minimum, or $1,140/yr.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The eligible household's $63 formula benefit is raised to NJ's $95/mo minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "After the standard, earned-income, and $395.49 property-tax shelter deductions, the formula benefit is $63/mo, and New Jersey's $95/mo minimum makes the annual benefit $1,140.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "With the $395.49 shelter deduction, net income is $805 (61% of FPL). The $63 formula benefit is raised to New Jersey's $95/mo state minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "The household is eligible with net income of $805, and its $63 formula benefit is raised to NJ's $95/mo minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-opus-5", + "old": "The $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The eligible household's $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "It left out the $395.49 property-tax excess shelter deduction and did not apply New Jersey's $95/mo minimum benefit, which sets this eligible household's benefit at $1,140/yr.", + "new": "It omitted the property-tax shelter deduction and NJ minimum used in the frozen calculation. The frozen $1,140 annual value assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "PolicyEngine finds that the household passes the SNAP work requirement test, so the household is eligible, and its $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "The frozen reference passes the work test because it assumes 40 weekly work hours. The prompt leaves hours unlisted and defaults them to zero; the developer decision recognizes that reading and excludes the output, so the ABAWD conclusion is not disproved by the $1,140 reference.", + "reason": "adjudication.json and reference_exclusions.json say the frozen reference assumed unlisted 40-hour weeks and recognize the zero-hours ABAWD reading rather than establishing eligibility." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "Only the $995.75/mo of property taxes counts, so the excess shelter deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It left out the $395.49 property-tax shelter deduction and did not apply New Jersey's $95/mo state minimum benefit for eligible households, which gives $1,140/yr.", + "new": "Its $0 explanation says 30% of net income exceeds the maximum allotment, without showing deductions or discussing the NJ minimum used in the frozen calculation. The frozen $1,140 annual value assumes 40 weekly work hours and property-tax-only shelter costs; work hours and occupied-home mortgage status are unlisted, so the output remains excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "Gross income of $1,486.54/mo is 112% of FPL, well under the limit, and the eligible household's $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Gross income of $1,486.54/mo is 112% of FPL, below the income limit. The frozen $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "Only the $995.75/mo of property taxes is a shelter cost, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95 minimum.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "Counted only self-employment and alimony income, leaving out the $12,000 of financial assistance, and gave the full $292/mo allotment. With the $1,000/mo of assistance counted as unearned income, net income is $805, the formula benefit is $63/mo, and New Jersey's $95 minimum sets the annual benefit at $1,140.", + "new": "Listed self-employment and alimony as countable income and returned an estimated maximum of $292/mo without explaining its treatment of the $12,000 financial assistance. Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted assumptions leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "The household is eligible (112% of FPL gross, 61% of FPL net), and its $63 formula benefit is raised to New Jersey's $95/mo state minimum, which the $0 answer ignores.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "Gave the full one-person maximum ($292 × 12), which requires zero net income. That result comes from leaving out the $1,000/mo of financial assistance and overstating shelter costs with mortgage interest. Correctly computed, net income is $805 and the $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Returned $3,504/yr, equal to $292 × 12, without a detailed income or shelter calculation. The release convention holds the FY2026 one-person maximum at $298/mo. Mortgage payments including interest count as SNAP shelter costs if the household occupies the mortgaged home; the frozen $1,140 reference instead assumes 40 weekly work hours and property-tax-only shelter costs, and remains excluded because work hours and occupied-home mortgage status are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "PolicyEngine finds that the household passes the SNAP work requirement test, so the household is eligible and receives New Jersey's $95/mo minimum benefit.", + "new": "The frozen reference passes the work test because it assumes 40 weekly work hours. The prompt leaves hours unlisted and defaults them to zero; the developer decision recognizes that reading and excludes the output, so the ABAWD conclusion is not disproved by the $1,140 reference.", + "reason": "adjudication.json and reference_exclusions.json say the frozen reference assumed unlisted 40-hour weeks and recognize the zero-hours ABAWD reading rather than establishing eligibility." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "glm-5.2", + "old": "Only the property taxes count, so the excess shelter deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95 minimum.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "The household passes the gross test (112% of FPL), the net test (61% of FPL), and the asset and work tests, and its $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "adjudication.json and reference_exclusions.json say the frozen reference assumed unlisted 40-hour weeks and recognize the zero-hours ABAWD reading rather than establishing eligibility." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "Monthly gross income of $1,486.54 is only 112% of FPL, the household is eligible, and its $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Monthly gross income of $1,486.54 is 112% of FPL, but the frozen $1,140 annual benefit also assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-5.5", + "old": "Correctly found the formula benefit below the minimum but applied the federal one-person minimum of about $23/mo. New Jersey's state-funded minimum SNAP benefit of $95/mo applies instead, which gives $1,140/yr.", + "new": "Applied about $23/mo as the federal minimum; c_snap_hold_fy2026 holds the federal minimum at $24/mo, while New Jersey's state minimum is $95/mo. The shown frozen formula benefit of $63.18 exceeds the federal minimum. The $1,140 annual reference remains excluded because work hours and occupied-home mortgage status are unlisted.", + "reason": "prompt.md shows a $63.18 formula benefit above the model's $23 federal minimum, latest_c_snap_hold_fy2026.py specifies the held $24 federal minimum, and reference_exclusions.json excludes the $1,140 reference." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It left out the $395.49 property-tax shelter deduction and did not apply New Jersey's $95/mo minimum benefit, which applies to this eligible household.", + "new": "Its $0 explanation attributes the result to high net income without showing deductions or discussing the NJ minimum used in the frozen calculation. The frozen $1,140 annual value assumes 40 weekly work hours and property-tax-only shelter costs; work hours and occupied-home mortgage status are unlisted, so the output remains excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "Only the $995.75/mo of property taxes counts, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95 minimum.", + "new": "Mortgage interest counts if the mortgage is on the occupied home; the sidecar recheck reproduces $1,908/yr with 40 weekly work hours and that shelter-cost reading. The frozen $1,140 reference uses different unlisted assumptions and remains excluded.", + "reason": "reference_exclusions.json permits occupied-home mortgage interest, and reference_outputs.csv.meta.json's 2026-09-29 recheck reproduces $1,908 with 40-hour weeks while retaining the exclusion." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-6-astra", + "old": "Only the $11,949 of property taxes counts, so the shelter deduction is $395.49/mo, the formula benefit is $63/mo, and New Jersey's $95 minimum gives $1,140/yr.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "Only the property taxes count, giving a $395.49/mo deduction, $805 net income, and a $63 formula benefit that New Jersey's $95/mo minimum raises to $1,140/yr.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "grok-4.3", + "old": "Gross income is 112% of FPL, New Jersey's BBCE removes the asset test, and the eligible household's $63 formula benefit is raised to NJ's $95/mo minimum.", + "new": "Gross income is 112% of FPL and New Jersey's BBCE removes the asset test. The frozen $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "grok-4.5", + "old": "The $995.75/mo of property taxes alone gives a $395.49 excess shelter deduction and a $63 formula benefit, and New Jersey's $95/mo minimum makes it $1,140/yr.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "grok-4.6", + "old": "which ignored the $11,949 of real estate taxes that are a homeowner shelter cost. It also did not apply New Jersey's $95/mo minimum benefit, which this eligible household receives.", + "new": "whereas the frozen calculation uses the real estate taxes as shelter costs. Its $1,140 annual benefit assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "grok-4.7", + "old": "With the $1,000/mo of assistance counted and the $395.49 property-tax shelter deduction, net income is $805, the formula benefit is $63, and New Jersey's $95 minimum gives $1,140/yr.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "inkling", + "old": "Only the $995.75/mo of property taxes counts, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95 minimum.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "minimax-m3", + "old": "Gross income is 112% of FPL and net income is 61% of FPL, both within limits, and the $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Both income tests pass in the frozen derivation. Its $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "ox-alpha", + "old": "Only the property taxes count, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to New Jersey's $95 minimum.", + "new": "The frozen reference uses property taxes alone, but mortgage interest counts if the mortgage is on the occupied home. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because work hours and mortgage residence are unlisted.", + "reason": "prior_amendments.json and reference_exclusions.json count mortgage payments including interest when the household occupies the mortgaged home, and adjudication.json excludes the output for that fact and unlisted work hours." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "The household passes both income tests, and its $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Under the frozen reference's assumed 40 weekly work hours and property-tax-only shelter costs, the formula benefit is $63/mo and New Jersey's $95 monthly minimum produces $1,140/yr; those unlisted inputs leave the output excluded from scoring.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_056__snap", + "field": "annotation", + "model": "qwen3.8-max", + "old": "Gross income of $1,486.54/mo is 112% of FPL and net income of $805 is 61% of FPL, and the eligible household's $63 formula benefit is raised to New Jersey's $95/mo minimum.", + "new": "Gross income of $1,486.54/mo is 112% of FPL and net income in the frozen derivation is 61% of FPL. Its $1,140 annual value additionally assumes 40 weekly work hours and property-tax-only shelter costs; those inputs are unlisted and the output is excluded.", + "reason": "reference_exclusions.json and adjudication.json exclude the $1,140 reference because work hours and occupied-home mortgage status are unlisted." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "The model correctly computed 3% × $10,506 = $315.18 using the unindexed $12,500 deduction instead of the 2026 inflation-adjusted $12,835. It then inflated the result to $496 based on an unspecified 'deduction structure' adjustment. The correct figure is 3% × $10,171.18 = $305.14.", + "new": "The model computed $315.18 using the $12,500 deduction, then raised it to $496 based on an unspecified 'deduction structure' adjustment. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "The model used Louisiana's statutory $12,500 single standard deduction without the inflation indexing that begins in 2026. That indexing raises the deduction to $12,835 and taxable income falls to $10,171.18, so the tax is $305.14, not $315.18.", + "new": "The model used the $12,500 deduction and flat 3% rate to obtain $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The model claimed Louisiana has no state individual income tax. In fact, Louisiana levies a flat 3% tax on AGI minus a $12,835 standard deduction in 2026, which gives $305.14 here.", + "new": "The model claimed Louisiana has no state individual income tax, although Louisiana levies a flat 3% tax. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "The model asserted that a large personal exemption plus the standard deduction wipes out the taxable base. Louisiana's post-reform system has only a $12,835 standard deduction and no separate exemption, which leaves $10,171.18 taxable at 3% = $305.14.", + "new": "The model asserted that a large personal exemption plus the standard deduction wipes out the taxable base. For this household, the frozen computation applies no separate exemption and leaves $10,171.18 taxable at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "The model applied the flat 3% rate to AGI minus the unindexed $12,500 deduction. Louisiana's deduction is inflation-adjusted to $12,835 for 2026, so taxable income is $10,171.18 and the tax is $305.14, not $315.", + "new": "The model applied the flat 3% rate to AGI minus the $12,500 deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "The model subtracted $12,500 from $23,006 and missed the 2026 inflation indexing of Louisiana's single standard deduction to $12,835. That overstated taxable income by $335 and gave $315 instead of $305.14.", + "new": "The model subtracted $12,500 from $23,006 and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "The model reached about $315 using the unindexed $12,500 deduction and then arbitrarily raised it to $446, citing capital-loss treatment. The $3,000 capital-loss cap is already reflected in the $23,006 AGI, and the correct deduction is $12,835, which gives $305.14.", + "new": "The model reached about $315 using the $12,500 deduction, then raised it to $446, citing capital-loss treatment. The $3,000 capital-loss cap is already reflected in its $23,006 AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "The model used the $12,500 base standard deduction rather than the 2026 inflation-indexed $12,835. Taxable income was overstated by $335 and the tax came out at $315.18 instead of $305.14.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "The model applied repealed pre-2025 Louisiana rules: a $4,500 personal exemption plus a deduction for federal income tax paid. Under the 2025 reform these were replaced by a single $12,835 standard deduction (2026), so taxable income is $10,171.18, not $17,705, and the tax is $305.14.", + "new": "The model subtracted a $4,500 personal exemption and an $801 federal-income-tax deduction from $23,006 AGI, leaving $17,705 taxable at 3% and about $531 of tax. Louisiana ended the federal income tax deduction after 2021; the 2025 reform replaced the personal exemption with a standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "prior_amendments.json records that Louisiana ended the federal income tax deduction after 2021, while reference_exclusions.json and adjudication.json classify $12,835/$305.14 as an excluded frozen computation." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The model computed about $315 using the unindexed $12,500 deduction and then added unspecified 'federal-based adjustments' to reach $448. Louisiana's 2026 computation is simply 3% × ($23,006.18 − $12,835) = $305.14.", + "new": "The model computed about $315 using the $12,500 deduction, then added unspecified 'federal-based adjustments' to reach $448. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "The model assumed the $12,500 standard deduction and ignored the 2026 inflation adjustment to $12,835. The result was $315.18 instead of 3% × $10,171.18 = $305.14.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "A taxable base of $7,606 means the model subtracted a $15,400 deduction from AGI. Louisiana's 2026 single standard deduction is $12,835, which leaves $10,171.18 taxable and a tax of $305.14.", + "new": "A taxable base of $7,606 means the model subtracted $15,400 from its $23,006 AGI, calling it the standard deduction without stating the amount. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "The model used the unindexed $12,500 Louisiana standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated taxable income by $335 and the tax by $10.04.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "The model used the repealed pre-2025 Louisiana system: a federal-income-tax deduction, a $4,500 deduction plus a $1,000 exemption, and 1.85%/3.5% graduated brackets. Since 2025 Louisiana taxes AGI minus a single standard deduction ($12,835 in 2026) at a flat 3%, which gives $305.14.", + "new": "The model used a federal-income-tax deduction that Louisiana ended after 2021, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "The model subtracted the base $12,500 standard deduction rather than the 2026 inflation-indexed $12,835. That gave $315.18 instead of $305.14.", + "new": "The model subtracted the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "The model subtracted the $8,389 ESI premiums from wages again, giving an AGI of $14,617 instead of $23,006. It then used the repealed $4,500 exemption and the 1.85% bracket. Under current law the tax is a flat 3% on AGI minus the $12,835 standard deduction, which gives $305.14.", + "new": "The model subtracted the $8,389 ESI premiums from wages, giving $14,617 AGI rather than the reference's $23,006.18, then used a repealed $4,500 exemption and 1.85% bracket. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "The model gave no derivation. $158 understates the tax and is consistent with outdated graduated rates or excess deductions. The correct computation is 3% × ($23,006.18 − $12,835) = $305.14.", + "new": "The model gave no derivation for $158, which is below the frozen reference and consistent with outdated rates or excess deductions. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The model deducted the ESI premiums from wages a second time, giving an AGI of $14,617. It then stacked a repealed $4,500 standard deduction and a $4,500 personal exemption and applied the repealed 1.85% bracket. The current flat 3% on $23,006.18 − $12,835 gives $305.14.", + "new": "The model used $14,617 AGI after subtracting ESI premiums, then stacked a $4,500 standard deduction and $4,500 personal exemption and applied the repealed 1.85% bracket. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "The model reduced AGI to $14,617 by subtracting the ESI premiums already excluded from wages, and used the repealed $4,500 exemption and 1.85% rate. Louisiana's 2026 tax is a flat 3% on AGI of $23,006.18 minus the $12,835 standard deduction, which is $305.14.", + "new": "The model used $14,617 AGI after subtracting ESI premiums and applied a repealed $4,500 exemption and 1.85% rate. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "The model claimed deductions and exemptions eliminate the tax. AGI of $23,006.18 exceeds Louisiana's $12,835 standard deduction by $10,171.18, which is taxed at 3% for $305.14.", + "new": "The model claimed deductions and exemptions eliminate the tax. The frozen computation leaves $10,171.18 taxable at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "The $315.18 answer reflects AGI minus the unindexed $12,500 deduction at 3%. The model missed the 2026 inflation adjustment that raises Louisiana's single standard deduction to $12,835, which yields $305.14.", + "new": "The $315.18 answer reflects $23,006 AGI minus the $12,500 deduction at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "The model submitted $152 while its explanation stated $130, and it gave no derivation for either figure. Both understate the tax, consistent with applying a repealed personal exemption or graduated-rate structure. The correct figure is 3% × ($23,006.18 − $12,835) = $305.14.", + "new": "The model submitted $152 while its explanation stated $130, with no derivation for either figure. Both are below the frozen reference. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "The model used the statutory $12,500 standard deduction without the 2026 inflation indexing to $12,835. Taxable income was overstated by $335, giving $315.18 instead of $305.14.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "The model applied the repealed 1.85% bottom-bracket rate instead of Louisiana's flat 3% rate. It also used the unindexed $12,500 deduction instead of $12,835. The correct tax is 3% × $10,171.18 = $305.14.", + "new": "The model applied the repealed 1.85% rate to AGI minus the $12,500 deduction. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "The model claimed taxable income falls below the lowest-bracket threshold. Louisiana has a flat 3% rate with no zero bracket, and the $12,835 standard deduction leaves $10,171.18 taxable, so the tax is $305.14.", + "new": "The model claimed taxable income falls below the lowest-bracket threshold. Louisiana has a flat 3% rate with no zero bracket, and the frozen computation leaves $10,171.18 taxable. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "The model used the $12,500 base single standard deduction instead of the 2026 inflation-indexed $12,835. That overstated the tax by $10.04 at $315.18.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "The model applied the unindexed $12,500 deduction. Louisiana's single standard deduction is inflation-adjusted to $12,835 for 2026, which makes taxable income $10,171.18 and the tax $305.14.", + "new": "The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "The model subtracted $12,500 rather than the 2026 indexed deduction of $12,835. The result was $315.18 instead of $305.14.", + "new": "The model subtracted $12,500 and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The model stacked a $4,500 personal exemption on top of a $12,500 standard deduction. The 2025 reform folded the exemption into a single standard deduction, which is $12,835 for 2026. Taxable income is $10,171.18, not $6,006, and the tax is $305.14.", + "new": "The model stacked a $4,500 personal exemption on top of a $12,500 deduction. The 2025 reform folded the exemption into a standard deduction; the frozen computation leaves $10,171.18 taxable at 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "That understated taxable income by $40 and the tax by $1.21.", + "new": "The $12,875 is the Department's provisional withholding and estimated-tax amount; its $40 difference from the frozen deduction explains $1.20 of the $1.21 tax gap, while the prompt's omitted $0.18 of wages and cent rounding explain the remainder. This output is excluded from scoring under d994.", + "reason": "prior_amendments.json records the provisional $12,875 amount and attributes $1.20 to the deduction difference and the remaining cent to omitted wage cents and rounding; adjudication.json excludes this output." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The model used the unindexed $12,500 standard deduction rather than the 2026 inflation-adjusted $12,835. The result was $315.18 instead of $305.14.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "The model applied the base $12,500 single deduction and missed the 2026 inflation indexing to $12,835. That overstated the tax by $10.04.", + "new": "The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "The model used $12,500 instead of the 2026 inflation-adjusted Louisiana standard deduction of $12,835. The result was $315.18 rather than $305.14.", + "new": "The model used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "The model used the repealed $4,500 exemption and the 1.85%/3.5% graduated brackets. Louisiana's 2025 reform replaced these with a flat 3% rate and a standard deduction of $12,835 in 2026, which gives $305.14.", + "new": "The model used the repealed $4,500 exemption and 1.85%/3.5% graduated brackets. Louisiana's 2025 reform replaced these with a flat 3% rate and a standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "The model used the unindexed $12,500 deduction rather than the 2026 indexed $12,835. Taxable income was overstated by $335, giving $315 instead of $305.14.", + "new": "The model used the $12,500 deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "The model got the flat 3% rate and the federal-tax-deduction repeal right but used the base $12,500 standard deduction. The correct 2026 inflation-indexed deduction is $12,835, which gives $305.14.", + "new": "The model got the flat 3% rate and federal-tax-deduction repeal right but used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-build-0.1", + "old": "The model re-subtracted the ESI premiums to get an AGI of $14,617, then applied the repealed $4,500 deduction and the 1.85% bracket. The correct base is $23,006.18 − $12,835 = $10,171.18, taxed at a flat 3% for $305.14.", + "new": "The model subtracted ESI premiums to obtain $14,617 AGI, then applied a repealed $4,500 deduction and 1.85% bracket. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "The model used the $12,500 statutory deduction without the 2026 inflation adjustment to $12,835. The result was $315 instead of $305.14.", + "new": "The model used the $12,500 deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k2.6", + "old": "The model returned no value or explanation for this output, so no answer could be parsed. The correct figure is 3% × ($23,006.18 − $12,835) = $305.14.", + "new": "The model returned no value or explanation for this output, so no answer could be parsed. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "The model applied the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated the tax at $315.18 versus $305.14.", + "new": "The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "The model falsely stated that Louisiana has no state income tax. Louisiana imposes a flat 3% tax on AGI minus a $12,835 standard deduction in 2026, which is $305.14 here.", + "new": "The model falsely stated that Louisiana has no state income tax; Louisiana imposes a flat 3% tax. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "The model subtracted the base $12,500 deduction and missed the 2026 inflation indexing to $12,835. That gave $315.18 instead of $305.14.", + "new": "The model subtracted the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "The model subtracted the $16,100 federal standard deduction plus a repealed $4,500 personal exemption. Louisiana uses its own $12,835 standard deduction with no separate exemption, so taxable income is $10,171.18, not $2,406, and the tax is $305.14.", + "new": "The model subtracted the $16,100 federal standard deduction plus a repealed $4,500 personal exemption. The frozen Louisiana computation applies its own standard deduction and no separate exemption. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "The model netted the full $7,968 capital loss against wages instead of applying the $3,000 loss cap, giving an AGI of $18,038 instead of $23,006. It then used the repealed $4,500 exemption and 1.85%/3.5% graduated brackets in place of the flat 3% rate on AGI minus the $12,835 standard deduction.", + "new": "The model netted the full $7,968 capital loss against wages instead of applying the $3,000 cap, giving $18,038 AGI, then applied a repealed $4,500 exemption and 1.85%/3.5% brackets. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "The model gave no derivation. $347.24 at 3% implies about $11,575 of taxable income, which means it subtracted roughly $11,431 instead of Louisiana's $12,835 2026 standard deduction. The correct computation is 3% × $10,171.18 = $305.14.", + "new": "The model gave no derivation; $347.24 at 3% implies about $11,575 taxable income and a deduction of roughly $11,431 from $23,006 AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994.", + "reason": "reference_exclusions.json (scenario_077, decision d994) and adjudication.json distinguish the excluded $12,835/$305.14 frozen computation from the published $12,838/$305.05 amounts." + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Louisiana's 2025 reform set a flat 3% rate and a single standard deduction that is inflation-indexed starting in 2026, so the 2026 deduction is $12,835 rather than the statutory base of $12,500. The tax is 3% × ($23,006.18 − $12,835) = $305.14. Most models used the correct structure with the unindexed $12,500 deduction, which gives $315.18. The other models relied on the repealed pre-2025 graduated brackets with personal exemptions and a federal-tax deduction, subtracted the ESI premiums from wages a second time, ignored the $3,000 capital-loss cap, or claimed Louisiana has no income tax.", + "new": "Louisiana's 2025 reform set a flat 3% rate and a $12,500 single standard deduction, indexed from 2026 under R.S. 47:294(B). PolicyEngine computed $12,835 from published CPI-U, giving the frozen reference of 3% × ($23,006.18 − $12,835) = $305.14. Louisiana first published its 2026 return deduction, $12,838, in RIB 26-019 on September 28, 2026; that gives $305.05, and this output is excluded from scoring under d994. Of the 48 models, 20 answered $315.18 or $315, amounts consistent with the $12,500 deduction and flat 3% rate; three more initially reached about $315 that way before raising their submitted answers to $446–$496. Other models used repealed graduated brackets or personal exemptions, deducted federal income tax despite that deduction ending after 2021, subtracted ESI premiums from wages, ignored the $3,000 capital-loss cap, or claimed Louisiana has no income tax.", + "reason": "reference_exclusions.json and adjudication.json record the d994 exclusion and published $12,838/$305.05 alternative, prior_amendments.json records the federal-tax deduction ending after 2021, and prompt.md shows 20 of 48 models submitted $315.18 or $315." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "It applied a $2,750 asset limit (also the wrong figure for a household with a disabled member) and denied SNAP.", + "new": "It applied a $2,750 asset limit and denied SNAP.", + "reason": "The kept exclusion record for scenario_080 SNAP (reference_exclusions.json) and the engine treat the household as having no SNAP elderly or disabled member, because the prompt states no qualifying route under 7 U.S.C. 2012(j); the aside calling it a household with a disabled member contradicts that, and the asset test is waived either way." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "so taxable income was about $260 too high (~$143,320 vs $143,059.47), and tax came out $63 high.", + "new": "so taxable income was about $260 above the frozen withholding-based reference (~$143,320 vs $143,059.47); its $26,995 is $3.69 above the recorded liability-based reading of $26,991.31.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "which dropped $4,127 of deductible SALT.", + "new": "which reduced SALT by $4,127 relative to the frozen reference's calculation using an unlisted withholding estimate.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "That put taxable income about $8,500 too high.", + "new": "That put taxable income about $8,500 above the frozen reference, which adds an unlisted withholding estimate; with no state income tax paid, the recorded sales-tax reading gives $28,763.81.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "Itemized deductions were therefore $245.71 low, which raised tax by about $60.", + "new": "Itemized deductions were $245.71 below the frozen reference, which uses an unlisted withholding estimate; its $26,992.66 is within $0.02 of the recorded corrected-liability reading of $26,992.64.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and prior_amendments.json record a corrected-liability reading of $26,992.64 rather than requiring the unlisted withholding estimate." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "understating itemized deductions by $4,127.", + "new": "putting itemized deductions $4,127 below the frozen reference, which uses an unlisted withholding estimate.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "so itemized deductions were $26,900 instead of $31,027.41 and taxable income was about $4,100 too high.", + "new": "so it itemized $26,900, about $4,100 below the frozen reference's $31,027.41, which uses an unlisted withholding estimate.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "left the $8,484.41 MA income tax out of SALT.", + "new": "put no state income tax in SALT, while the frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "so taxable income was about $950 too high ($144,010 vs $143,059).", + "new": "so taxable income was about $950 above the frozen withholding-based reference ($144,010 vs $143,059), and about $705 above the recorded liability-based reading ($143,305.47).", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "It took a $15,000 standard deduction instead of itemizing $31,027.41 ($16,900 mortgage interest + $5,643 real estate tax + $8,484.41 estimated MA tax). That left taxable income about $16,000 too high.", + "new": "It took a $15,000 standard deduction even though mortgage interest plus real estate tax alone total $22,543. The frozen reference itemizes $31,027.41 using an unlisted MA withholding estimate; the recorded zero-state-income-tax reading itemizes $23,396. Its taxable income was about $16,000 above the frozen reference.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "left the $8,484.41 MA income tax out of SALT, itemizing only $22,543.", + "new": "itemized $22,543 with no state income tax in SALT; the frozen reference instead adds an $8,484.41 estimate of MA withholding that the prompt does not list.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "Its taxable income of $169,008 drops nearly all of the $31,027.41 in itemized deductions; the correct figure is $143,059.47. Its $24,197 tax also does not follow from 2026 single brackets applied to that base, which would give about $33,160.", + "new": "Its taxable income of $169,008 leaves out most itemized deductions: $31,027.41 in the frozen reference, using an unlisted withholding estimate, or $23,396 under the recorded zero-state-income-tax reading. The frozen reference's taxable income is $143,059.47. Its $24,197 tax also does not follow from 2026 single brackets applied to its $169,008 base, which would give about $33,160.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "below the correct $143,059.47.", + "new": "below the frozen reference's $143,059.47, which uses an unlisted MA withholding estimate.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json records $26,932.27 as a frozen value dependent on unlisted withholding, and adjudication.json excludes it from scoring." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It correctly estimated about $8,238 of MA income tax but then capped SALT at the old $10,000 instead of the 2026 $40,400 cap. That limited itemized deductions to $26,900 instead of $31,027.41.", + "new": "It estimated about $8,238 of MA income tax, close to the excluded frozen liability of $8,238.41; the recorded Part A loss-offset correction gives $8,232.90. It then capped SALT at the old $10,000 instead of the 2026 $40,400 cap, itemizing $26,900. The frozen federal reference itemizes $31,027.41 using an unlisted withholding estimate.", + "reason": "The scenario_081 state exclusion in reference_exclusions.json and its 2026-10-13 sidecar recheck retain $8,238.41 as excluded and record $8,232.90 after the Part A correction, while the federal exclusion identifies the unlisted withholding estimate." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "under which the correct tax is $26,932.27.", + "new": "under which the frozen reference's calculation, using an unlisted withholding estimate, yields $26,932.27.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json records $26,932.27 as a frozen value dependent on unlisted withholding, and adjudication.json excludes it from scoring." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "which overstated taxable income by about $100.", + "new": "which put taxable income about $100 above the frozen reference, whose withholding estimate is an unlisted input.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "which give $26,932.27.", + "new": "which give the frozen reference's $26,932.27 when its unlisted withholding estimate is used.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json records $26,932.27 as a frozen value dependent on unlisted withholding, and adjudication.json excludes it from scoring." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It used MA tax liability (about $8,238) as the SALT income-tax component instead of PolicyEngine's withholding estimate of 5% × ($174,088.12 − $4,400) = $8,484.41. That understated itemized deductions by about $250 and overstated tax by $60.", + "new": "Its answer is consistent with using its MA liability estimate of $8,238.70 in SALT; its federal explanation gives no state-tax figure. The frozen reference instead uses an unlisted withholding estimate of $8,484.41. Its $26,992.59 is within $0.05 of the recorded corrected-liability reading of $26,992.64.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and prior_amendments.json record a corrected-liability reading of $26,992.64 rather than requiring the unlisted withholding estimate." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "so the deduction was $254.31 short and tax came out about $61 high.", + "new": "so the deduction was $254.31 below the frozen reference, which uses an unlisted withholding estimate; its $26,993.29 is within $1 of the recorded corrected-liability reading of $26,992.64.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and prior_amendments.json record a corrected-liability reading of $26,992.64 rather than requiring the unlisted withholding estimate." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "Taxable income was $154 too high ($143,213.70) and tax was $37 high.", + "new": "Taxable income was $154 above the frozen withholding-based reference ($143,213.70), and its $26,969.29 was $37.02 above that reference but $22.02 below the recorded liability-based reading of $26,991.31.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "That understated SALT by $254 and overstated tax by about $61.", + "new": "That put SALT $254 below the frozen reference, which uses an unlisted withholding estimate; its $26,993.29 is within $1 of the recorded corrected-liability reading of $26,992.64.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and prior_amendments.json record a corrected-liability reading of $26,992.64 rather than requiring the unlisted withholding estimate." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "so the correct base is $143,059.47 taxed at 10/12/22/24%.", + "new": "so the 10/12/22/24% brackets apply; the $143,059.47 base belongs to the frozen reference's calculation using an unlisted withholding estimate.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json records $26,932.27 as a frozen value dependent on unlisted withholding, and adjudication.json excludes it from scoring." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-build-0.1", + "old": "left the $8,484.41 MA income tax out of SALT and itemized only $22,543.", + "new": "itemized $22,543 with no state income tax in SALT; the frozen reference instead adds an $8,484.41 estimate of MA withholding that the prompt does not list.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "It itemized only $22,543 (mortgage interest plus real estate tax) and left out the $8,484.41 estimated MA income tax that belongs in SALT under the 2026 $40,400 cap. Taxable income was $151,543.80 instead of $143,059.47.", + "new": "It itemized $22,543 (mortgage interest plus real estate tax), giving taxable income of $151,543.80. The frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list, giving taxable income of $143,059.47; with no state income tax paid, the recorded SALT reading instead adds $853 of sales tax.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "which overstated taxable income by about $108.", + "new": "which put taxable income about $108 above the frozen reference, whose withholding estimate is an unlisted input.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "Itemized deductions were overstated by about $54, so tax came out $11 low.", + "new": "Itemized deductions were about $54 above the frozen reference, whose withholding estimate is an unlisted input; its tax was $11.27 below that reference and $70.31 below the recorded liability-based reading of $26,991.31.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "it also left the $8,484.41 MA tax out of SALT.", + "new": "it also put no state income tax in SALT, while the frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "adding the $8,484.41 MA income tax to SALT brings itemized deductions to $31,027.41.", + "new": "the frozen reference adds an $8,484.41 estimate of MA withholding that the prompt does not list, bringing itemized deductions to $31,027.41.", + "reason": "The scenario_081 federal exclusion in reference_exclusions.json and the SALT corrections in prior_amendments.json distinguish the unlisted withholding estimate from liability and zero-payment readings." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "It also stated wrongly that Massachusetts has no employee payroll tax, so it missed the $805.01 PFML employee contribution at 0.46%.", + "new": "It said Massachusetts has no separate mandatory employee payroll tax and left out PFML; the reference assumes the maximum $805.01 deduction at 0.46%, but the employer may deduct less or none.", + "reason": "prompt.md records a claim about mandatory employee payroll tax, and adjudication.json and the payroll_tax entry in reference_exclusions.json accept federal tax alone under that reading." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It got the PFML rate wrong, claiming employees pay 0% of the medical portion, so it set PFML to ~$0 instead of $805.01 at 0.46%.", + "new": "It estimated PFML at about $0; the reference assumes the maximum deductible employee share of 0.46%, or $805.01, while the employer may deduct less or none.", + "reason": "The payroll_tax entry in reference_exclusions.json and adjudication.json establish that $805.01 is the maximum deductible share rather than a required employee contribution, as prior_amendments.json previously corrected for this model." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It reached the correct $14,192.66 using 0.46%,", + "new": "It reached the frozen reference of $14,192.66 using the maximum deductible 0.46% share,", + "reason": "adjudication.json records $14,192.66 as an excluded frozen reference and $13,387.65 as the alternative, rather than declaring the maximum-withholding total uniquely correct." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "The real 2026 split is 0.18% family and 0.70% medical, and employees pay all of the family portion plus 40% of the medical portion.", + "new": "The reference uses published 2026 rates of 0.18% family and 0.70% medical; the employer may deduct up to all of the family portion plus 40% of the medical portion, or deduct less or none.", + "reason": "The payroll_tax entry in reference_exclusions.json describes these percentages as maximum permitted employer deductions, not amounts employees must pay, matching the correction in prior_amendments.json." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "It used a stale 0.375% PFML employee rate ($656.26) instead of the 2026 rate of 0.46% ($805.01).", + "new": "It counted PFML at 0.375% ($656.26), below the reference's maximum deductible 0.46% share ($805.01); the employer may deduct less or none.", + "reason": "The payroll_tax entry in reference_exclusions.json and adjudication.json identify 0.46% as the reference's maximum deductible share and leave the employer's actual withholding unspecified." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "Applying 0.46% to $175,002 gives a PFML contribution of $805.01, so its PFML amount was far too small.", + "new": "The reference assumes the maximum deductible PFML share of 0.46% × $175,002 = $805.01; its implied PFML amount is below that maximum, but the employer may deduct less or none.", + "reason": "The payroll_tax entry in reference_exclusions.json permits smaller deductions, and prior_amendments.json previously replaced this model's underpayment claim with a comparison to the reference maximum." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "glm-5.2", + "old": "For 2026 the employee medical share is 40% of 0.70% = 0.28%, so the total employee rate is 0.46% and the contribution is $805.01, not $931.", + "new": "The reference uses the maximum deductible medical share of 40% of 0.70% = 0.28%, plus 0.18% family, totaling 0.46% and $805.01; the model's $931 exceeds that maximum, and the employer may deduct less or none.", + "reason": "The payroll_tax entry in reference_exclusions.json identifies these shares as maximum permitted deductions, and prior_amendments.json previously qualified this model's comparison as the engine's maximum-withholding assumption." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "minimax-m3", + "old": "The correct figure is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 MA PFML.", + "new": "The frozen reference is $10,850.12 Social Security + $2,537.53 Medicare + $805.01 MA PFML, assuming maximum PFML withholding; counting only mandatory employee contributions gives federal tax alone, $13,387.65.", + "reason": "adjudication.json and the payroll_tax entry in reference_exclusions.json exclude the frozen reference because maximum PFML withholding is an unstated employer choice, repeating the issue corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_081__payroll_tax", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "denied any MA payroll tax", + "new": "said no mandatory state payroll taxes apply in Massachusetts", + "reason": "prompt.md records a denial of mandatory state payroll taxes, which adjudication.json and the payroll_tax entry in reference_exclusions.json distinguish from optional PFML withholding." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "case_annotation", + "old": "The other wrong answers came from outdated or uncapped PFL rates, dropping either PFL or DBL, or wrongly taking ESI premiums or 401(k) deferrals out of FICA wages.", + "new": "Other answers differed from the frozen reference by using outdated or uncapped PFL rates, including only PFL or DBL, or reducing the engine's FICA wage base by ESI premiums or 401(k) deferrals; omitting either optional New York contribution is not itself a model error.", + "reason": "adjudication.json and prior_amendments.json record that the employer may collect less or none of either New York contribution, so omitting either share is not inherently a model error." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "Correctly identified DBL at $31.20 but contradicted itself on PFL and settled on an unsupported $49.31. The correct PFL amount is the $411.91 cap (0.432% × $100,195 = $432.84, above the cap), so its NY component of $80.51 should be $443.11.", + "new": "Included DBL at its $31.20 maximum but contradicted itself on PFL and settled on an unsupported $49.31. The reference counts the maximum the employer may collect: $411.91 of PFL (0.432% × $100,195 = $432.84, capped) plus $31.20 of DBL, or $443.11, against the model's $80.51; the employer may collect less or none.", + "reason": "adjudication.json, reference_exclusions.json and the prior claude-opus-4.7 amendment identify these amounts as optional maximum collectible shares rather than the household's uniquely correct contributions." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "Included DBL ($31.20) correctly but used the 2025 PFL parameters (0.388% × an $89,343 wage cap = $346.65). The 2026 parameters are 0.432% with a $411.91 maximum, so it understated PFL by $65.26.", + "new": "Included DBL at its $31.20 maximum but used the 2025 PFL rate of 0.388% with a cap of about $89,343, yielding $346.65. The reference counts the maximum collectible PFL share at the 2026 rate of 0.432%, capped at $411.91, so the model's PFL amount is $65.26 below the reference; the employer may collect less or none.", + "reason": "adjudication.json identifies the optional maximum-share assumption, while prior_amendments.json and reference_audit/2026-10-05-payroll/verification/law_research.json record the 2025 PFL maximum as $354.53 rather than $346.65." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The 2026 amounts are PFL at the $411.91 cap (0.432% × $100,195 = $432.84, capped) plus DBL at $31.20, for $443.11.", + "new": "The reference counts the maximum the employer may collect: PFL at the $411.91 cap (0.432% × $100,195 = $432.84, capped) plus DBL at $31.20, for $443.11; the employer may collect less or none.", + "reason": "adjudication.json and the prior claude-sonnet-5 amendment distinguish the reference's maximum collectible shares from the household's actual contributions." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "PFL has the same employee-contribution status as DBL and adds $411.91 (0.432% of wages, capped).", + "new": "The reference also includes PFL at the maximum the employer may collect, $411.91 (0.432% of wages, capped); the employer may collect less or none of either contribution.", + "reason": "adjudication.json and the prior deepseek-v4-pro amendment establish that collection of either employee share is optional." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "Used about $333 for NY PFL, an outdated maximum; the 2026 amount is $411.91 (0.432% rate, capped). It also rounded DBL to $31, so the NY component of $364 should be $443.11.", + "new": "Used about $333 for NY PFL and rounded DBL to $31, giving about $364 of NY contributions. The reference counts the maximum the employer may collect in 2026, $411.91 of PFL (0.432% rate, capped) plus $31.20 of DBL, or $443.11; the employer may collect less or none.", + "reason": "adjudication.json and reference_exclusions.json identify $443.11 as the reference's optional maximum collectible share rather than the household's required contribution." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "It also used ~$394 for PFL instead of $411.91 and left out DBL's $31.20.", + "new": "It also used ~$394 for PFL and no DBL, compared with the reference's maximum collectible shares of $411.91 and $31.20; the employer may collect less or none.", + "reason": "adjudication.json and the prior gemini-3.5-flash amendment establish that the employee shares are optional for the employer to collect." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "It also put about $270 on NY SDI/PFL when the correct amount is $443.11 ($411.91 PFL cap + $31.20 DBL).", + "new": "It also put about $270 on NY SDI/PFL, below the reference's $443.11 maximum collectible share ($411.91 PFL cap + $31.20 DBL); the employer may collect less or none.", + "reason": "adjudication.json and the prior gemini-3.6-flash amendment identify $443.11 as an optional maximum collectible share rather than the household's correct contribution." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "It also used about $334 for NY SDI/PFL instead of $443.11.", + "new": "It also used about $334 for NY SDI/PFL, below the reference's $443.11 maximum collectible share; the employer may collect less or none.", + "reason": "adjudication.json and the prior gemini-3.7-flash amendment establish that the reference counts maximum shares the employer is permitted, but not required, to collect." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "glm-5.2", + "old": "Included DBL ($31.20) correctly but computed PFL with the 2025 rate (0.388% × ~$96,200 = $373.26). The 2026 rate of 0.432% hits the $411.91 maximum, so it understated PFL by $38.65.", + "new": "Included DBL at its $31.20 maximum but computed PFL with the 2025 rate (0.388% × ~$96,200 = $373.26). The reference counts the maximum collectible PFL share at the 2026 rate of 0.432%, capped at $411.91, so the model's PFL amount is $38.65 below the reference; the employer may collect less or none.", + "reason": "adjudication.json and reference_exclusions.json identify the reference's maximum collectible shares rather than mandatory household contributions." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "gpt-5.5", + "old": "The 2026 amounts are PFL at the $411.91 cap (0.432% rate) plus DBL at $31.20, for $443.11.", + "new": "The reference counts the maximum the employer may collect in 2026: PFL at the $411.91 cap (0.432% rate) plus DBL at $31.20, for $443.11; the employer may collect less or none.", + "reason": "adjudication.json and reference_exclusions.json establish that $443.11 is the optional maximum collectible share rather than this household's actual contribution." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "PFL adds $411.91: 0.432% of $100,195 is $432.84, capped at $411.91.", + "new": "The reference also counts the maximum collectible PFL share of $411.91: 0.432% of $100,195 is $432.84, capped at $411.91; the employer may collect less or none of either contribution.", + "reason": "adjudication.json and the prior gpt-5.6-luna amendment establish that collection of both employee shares is optional." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "inkling", + "old": "It also rounded SDI to $31, so the NY component of $396 should be $443.11.", + "new": "It also rounded SDI to $31, giving about $396 of NY contributions against the reference's $443.11 maximum collectible share; the employer may collect less or none.", + "reason": "adjudication.json and reference_exclusions.json identify $443.11 as the optional maximum collectible share rather than this household's required contribution." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "muse-wonder", + "old": "It also used $372.40 for NY contributions instead of $443.11.", + "new": "It also used $372.40 for NY contributions, below the reference's $443.11 maximum collectible share; the employer may collect less or none.", + "reason": "adjudication.json and reference_exclusions.json establish that $443.11 is the reference's optional maximum collectible share rather than this household's actual required contribution." + }, + { + "case_id": "us__scenario_082__payroll_tax", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "Correctly saw that NY PFL and SDI add to FICA, then subtracted an invented ~$200 \"NY deduction\" from $7,664.92 and reported $7,464.33. The correct NY addition is +$443.11 ($411.91 PFL cap + $31.20 DBL).", + "new": "At one point added NY PFL and SDI to FICA, then subtracted an invented ~$200 \"NY deduction\" from $7,664.92 and reported $7,464.33. The reference adds the maximum the employer may collect, $443.11 ($411.91 PFL cap + $31.20 DBL); the employer may collect less or none.", + "reason": "adjudication.json and the prior qwen-3.7-max amendment distinguish the reference's optional maximum collectible share from a required NY addition." + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "roughly double the correct bracket tax", + "new": "about $2,860 above the $5,239.62 bracket tax at that income", + "reason": "The held joint schedule in latest_c_ca_hold_2025.py, as the prior qwen3.8-max amendment records, gives $5,239.62 on $130,894, so $8,100 is about 55% above that tax." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-fable-5", + "old": "It also subtracted a $94 medical deduction and no $453.64 excess shelter deduction, so net income came out at $89 instead of $237.09.", + "new": "It also subtracted a $94 medical deduction, giving $89 net income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It replaced the $453.64/month excess shelter deduction with a $1,124/yr medical deduction, which gave $86.67/month net instead of $237.09 and an inflated $9,108.", + "new": "It took a $1,124/yr medical deduction and no shelter deduction, giving $86.67/month net and $9,108. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct calculation is the $785 maximum minus 30% of $237.09 net income, about $714/month, or $8,625.89 for the year.", + "new": "The record's r30_snap_heat_and_eat_sua correction gives $691 net, a rounded $208 contribution and $577/month ($6,924/year), using the FY2026 $785 maximum held all year. The excluded frozen reference's $8,625.89 includes the defect-based shelter deduction and projected FY2027 October–December parameters.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "net income is 11% of FPG, the $2,800 in assets is within limits, and TANF non-cash categorical eligibility applies. A near-maximum allotment of about $714/month is payable.", + "new": "the $2,800 in assets is within limits, and TANF non-cash categorical eligibility applies. The record's r30_snap_heat_and_eat_sua correction gives $691 net income and $577/month; about $714/month is the frozen reference's January–September amount, which includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It never added the roughly $505/month of non-wage income or the $453.64 excess shelter deduction that produce the $237.09 net income.", + "new": "It never added the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "The model's net income of about $194/month counts wages only and leaves out the roughly $505/month of non-wage benefit income and the $453.64 excess shelter deduction. It used a $766 maximum allotment instead of $785 (rising to $802.60 for October–December) and then made an unexplained cut from $8,496 to $8,316.", + "new": "The model's net income of about $194/month counts wages only and leaves out the roughly $505/month of non-wage benefit income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month. It used a $766 maximum allotment instead of the FY2026 $785 held for all 12 months; $802.60 is the frozen engine's October–December projection. It then made an unexplained cut from $8,496 to $8,316.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-5", + "old": "The model assumed net income was just wages less the earned income and standard deductions, leaving out the roughly $505/month of non-wage income and the $453.64 shelter deduction. It rounded to $8,300 instead of computing $785 − $71.10 for January–September plus the higher $802.60 maximum for October–December.", + "new": "The model assumed net income was just wages less the earned income and standard deductions, leaving out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net, a $208 contribution and $577/month. Its $8,300 is below its own wages-only formula; holding the FY2026 $785 maximum for all 12 months matches c_snap_hold_fy2026, whereas the frozen reference's $71.10 contribution and October–December $802.60 maximum reflect the shelter defect and engine projection.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 excess shelter deduction, which together give $237.09 net and a $71.10 contribution instead of about $27.60.", + "new": "It left out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 contribution rather than its approximately $27.60; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "The model treated wages as the only income and subtracted a $93.67 medical deduction with no shelter deduction, getting $72.67 net instead of $237.09.", + "new": "The model treated wages as the only income and subtracted a $93.67 medical deduction, getting $72.67 net. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; $237.09 is the frozen reference's defect-based net.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It missed the roughly $505/month of non-wage benefit income and the $453.64 shelter deduction that leave $237.09 net and a $71.10 monthly contribution.", + "new": "It missed the roughly $505/month of non-wage benefit income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "It double-counted the $564 premium in a medical deduction, left out the roughly $505/month of non-wage income and the $453.64 shelter deduction, and used a $797 maximum, giving net income of $120 instead of $237.09.", + "new": "It double-counted the $564 premium in a medical deduction, left out the roughly $505/month of non-wage income, and used a $797 maximum, giving $120 net income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net; $237.09 is the frozen reference's defect-based net.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "The model correctly added TANF income ($458.54/month) to wages but left out the $453.64/month excess shelter deduction, so net income was $602.21 instead of $237.09.", + "new": "The model included estimated TANF income ($458.54/month) with wages and obtained $602.21 net income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "It added a $140.67 medical deduction (double-counting the premium) in place of the $453.64 shelter deduction and used a $768 maximum.", + "new": "It added a $140.67 medical deduction (double-counting the premium) and used a $768 maximum. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 excess shelter deduction, and used an $805 maximum instead of $785 (January–September).", + "new": "It left out the roughly $505/month of non-wage income and used an $805 maximum instead of the FY2026 $785 held for all 12 months. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "It missed the roughly $505/month of non-wage income and the $453.64 shelter deduction that give $237.09 net and a $71.10 contribution.", + "new": "It missed the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "Net income is $237.09, which reduces the allotment by $71.10/month.", + "new": "The record's r30_snap_heat_and_eat_sua correction gives $691 net income and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The correct result is $785 − 0.3×$237.09 ≈ $714/month for January–September and about $733.60 for October–December.", + "new": "The record's r30_snap_heat_and_eat_sua correction gives $691 net, a rounded $208 contribution and $577/month, with the FY2026 $785 maximum held all year. The frozen reference's approximately $714 and $733.60 monthly amounts include the shelter defect and projected FY2027 October–December parameters.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "The model included TANF as unearned income but left out the $453.64/month excess shelter deduction, which pushed net income far above $237.09.", + "new": "The model included TANF as unearned income and took no utility-allowance shelter deduction, as the record's r30_snap_heat_and_eat_sua correction does. That correction gives $691 net and $577/month; $237.09 is the frozen reference's net, which includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "so a benefit of about $714/month is payable.", + "new": "so SNAP is positive under the record's stated-facts correction: $577/month after correcting r30_snap_heat_and_eat_sua; about $714/month is the frozen reference's January–September amount, which includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 shelter deduction, which understated the $237.09 net income and overstated the benefit at $9,036.", + "new": "It left out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $6,924/year; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction, and the model's $9,036 is above both annual comparisons.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "the roughly $505/month of non-wage income that raises net income to $237.09.", + "new": "the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 contribution; $237.09 is the frozen reference's net, which includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 shelter deduction, and used a $768 maximum instead of $785/$802.60.", + "new": "It left out the roughly $505/month of non-wage income and used a $768 maximum instead of the FY2026 $785 held for all 12 months. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction and $802.60 October–December maximum come from the defect and engine projection.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "glm-5.3", + "old": "The model's $179.34 net income uses wages only, with no non-wage benefit income (~$505/month) and no $453.64 shelter deduction. It paired that with an $800 maximum for every month instead of $785 for January–September and $802.60 for October–December.", + "new": "The model's $179.34 net income uses wages only and omits non-wage benefit income (~$505/month). Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect. It used an $800 maximum for every month instead of the FY2026 $785 held all year under c_snap_hold_fy2026; $802.60 is the frozen engine's October–December projection.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "with a $237.09 net income and about $714/month benefit.", + "new": "with $691 net income and $577/month under the record's r30_snap_heat_and_eat_sua correction; $237.09 net and about $714/month are the frozen reference's January–September figures, which include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It left out the roughly $505/month of non-wage income that leaves $237.09 net after deductions and reduces the allotment by $71.10/month.", + "new": "It left out the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The model counted benefit income (SSI) in SNAP income but left out the $453.64/month excess shelter deduction, so net income was well above $237.09. That produced about $533/month instead of about $714.", + "new": "The model counted benefit income as SSI, although the record has SSI at $0 and roughly $505/month of TANF. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month. Its approximately $533/month is below that comparison; the frozen reference's approximately $714/month includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It left out the roughly $505/month of non-wage income; net income is $237.09, which reduces each month's allotment by $71.10.", + "new": "It left out the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The model computed a single $710.92 monthly amount from wages alone with no shelter deduction and multiplied it by 12. That misses the October–December months, when the maximum rises to $802.60 and the benefit is about $733.60. The correct monthly benefit for January–September is $785 − $71.10, from $237.09 net income that includes about $505 of non-wage income less a $453.64 shelter deduction.", + "new": "The model computed a single $710.92 monthly amount from wages alone with no shelter deduction and multiplied it by 12. Holding the FY2026 schedule all year matches c_snap_hold_fy2026, and taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction. With roughly $505/month of TANF counted, that correction gives $691 net, a $208 contribution and $577/month. The frozen reference's $237.09 net, $71.10 contribution and October–December $733.60 benefit on a $802.60 maximum reflect the shelter defect and engine projection.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-sol", + "old": "The model included estimated TANF cash income but applied only the earned income and standard deductions and left out the $453.64/month excess shelter deduction. Net income was therefore far above $237.09, and the benefit came out at $618/month instead of about $714.", + "new": "The model included estimated TANF cash income and applied only the earned income and standard deductions, as the record's r30_snap_heat_and_eat_sua correction does. That correction gives $691 net, a $208 contribution and $577/month, below its $618/month; the frozen reference's $237.09 net and approximately $714/month include the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "The model explicitly set the shelter deduction to zero, dropping the $453.64/month excess shelter deduction, while counting TANF income. Net income was overstated, giving about $647/month instead of $713.90 ($785 − $71.10).", + "new": "The model explicitly set the shelter deduction to zero while counting TANF income, as the record's r30_snap_heat_and_eat_sua correction does. That correction gives $691 net, a $208 contribution and $577/month, below its approximately $647/month; the frozen reference's $713.90 ($785 − $71.10) includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.3", + "old": "The household is income-, asset- and categorically eligible, with $237.09 net income and a benefit of about $714/month.", + "new": "The household is income-, asset- and categorically eligible under the record's stated-facts reading, whose r30_snap_heat_and_eat_sua correction gives $691 net and $577/month; $237.09 net and about $714/month are the frozen reference's January–September figures, which include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.5", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 shelter deduction, and used a $766 maximum instead of $785.", + "new": "It left out the roughly $505/month of non-wage income and used a $766 maximum instead of $785. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.6", + "old": "It ignored the roughly $505/month of non-wage income that leaves $237.09 net after the $209 standard, $98.58 earned income, and $453.64 shelter deductions, and it used $766 instead of $785.", + "new": "It ignored the roughly $505/month of non-wage income and used $766 instead of $785. The record's r30_snap_heat_and_eat_sua correction subtracts the $209 standard and $98.58 earned income deductions but no utility-allowance shelter deduction, giving $691 net and a $208 contribution; the frozen reference's $237.09 net includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.7", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 shelter deduction, and used a $787/month maximum.", + "new": "It left out the roughly $505/month of non-wage income and used a $787/month maximum. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction; the frozen reference's $453.64 shelter deduction comes from that defect.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-build-0.1", + "old": "It also assumed zero net income, ignoring the roughly $505/month of non-wage income that leaves $237.09 net and a $71.10 contribution.", + "new": "It also assumed zero net income, ignoring the roughly $505/month of non-wage income. The record's r30_snap_heat_and_eat_sua correction gives $691 net and a $208 contribution; the frozen reference's $237.09 net and $71.10 contribution include the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "inkling", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 shelter deduction that give $237.09 net and a monthly benefit of about $714.", + "new": "It left out the roughly $505/month of non-wage income. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and $577/month; the frozen reference's $237.09 net and about $714/month include the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "kimi-k2.6", + "old": "there was no answer to score against the $8,625.89 annual benefit.", + "new": "there was no usable answer to compare with the excluded frozen reference's $8,625.89 annual value.", + "reason": "reference_exclusions.json retains $8,625.89 as the excluded frozen value for scenario_100 SNAP, and adjudication.json removes this output from scoring for every model." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "minimax-m3", + "old": "The correct figure is $785 − 0.3×$237.09 ≈ $714.", + "new": "The record's r30_snap_heat_and_eat_sua correction gives $691 net and a rounded $208 contribution, so its monthly figure is $785 − $208 = $577; about $714 is the frozen reference's January–September figure, which includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "muse-wonder", + "old": "It left out the roughly $505/month of non-wage income and the $453.64 shelter deduction, so the contribution was about $29 instead of $71.10.", + "new": "It left out the roughly $505/month of non-wage income, giving a contribution of about $29. Taking no utility-allowance shelter deduction matches the record's r30_snap_heat_and_eat_sua correction, which gives $691 net and a $208 contribution; $71.10 is the frozen reference's contribution, which includes the defect-based $453.64 shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "ox-alpha", + "old": "Combined with leaving out the roughly $505/month of non-wage income, this drove net income to $3 instead of $237.09.", + "new": "Combined with leaving out the roughly $505/month of non-wage income, this drove net income to $3. The record's r30_snap_heat_and_eat_sua correction gives $691 net and $577/month; $237.09 is the frozen reference's net, which includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "qwen3.8-max", + "old": "so it receives about $714/month.", + "new": "so SNAP is positive under the record's stated-facts correction: $577/month after correcting r30_snap_heat_and_eat_sua; about $714/month is the frozen reference's January–September amount, which includes the defect-based shelter deduction.", + "reason": "reference_outputs.csv.meta.json revisions[17] and [19] give $691 net, a $208 contribution and $577/month after correcting r30_snap_heat_and_eat_sua, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "case_annotation", + "old": "Allowable deductions are the $209 standard deduction, the $98.58 earned income deduction (20% of wages) and a $453.64 excess shelter deduction. There is no excess medical deduction, so net income is $237.09/month and the household contribution is $71.10/month. The benefit is $785 − $71.10 ≈ $713.90 for January–September and about $733.60 for October–December, when the maximum allotment rises to $802.60, for $8,625.89 a year. Most models counted wages only, added a medical deduction and no shelter deduction, and pushed net income near zero, so they landed near the maximum (~$9,000–9,420). Models that did add TANF income left out the $453.64 shelter deduction, so they came in too low.", + "new": "The frozen reference subtracts the $209 standard deduction, the $98.58 earned income deduction and a $453.64 excess shelter deduction, with no medical deduction, giving $237.09 net and a $71.10 contribution. Its approximately $713.90 January–September and $733.60 October–December monthly amounts aggregate to the frozen annual value of $8,625.89; the latter uses the engine's projected FY2027 $802.60 maximum. The publication convention c_snap_hold_fy2026 holds the FY2026 $785 maximum and schedule all year; with the rounding fixes, the earlier engine gave $713/month ($8,556/year). The $453.64 shelter deduction comes from r30_snap_heat_and_eat_sua. Correcting that defect on the record's stated-facts reading takes no utility-allowance shelter deduction and gives $691 net, a $208 contribution and $577/month ($6,924/year), now reproduced by policyengine-us 2.41.0. Many models counted wages only and added a medical deduction, landing near $9,000–9,420. Taking no shelter deduction agrees with the recorded correction; the TANF-counting models came in above its $6,924, although most were below the frozen reference. Neither disability reading is established, so the output remains excluded.", + "reason": "reference_outputs.csv.meta.json revisions[0] holds FY2026 all year, and revisions[17] and [19] remove the r30 shelter deduction and give $691 net and $577/month, while reference_exclusions.json retains the disability-input exclusion." + }, + { + "case_id": "us__scenario_100__snap", + "field": "reasoning", + "old": "r30_snap_heat_and_eat_sua (engine defect, not fixed upstream)", + "new": "r30_snap_heat_and_eat_sua (engine defect, since fixed upstream by PolicyEngine/policyengine-us#10033, which the policyengine-us 2.41.0 references include)", + "reason": "reference_outputs.csv.meta.json revisions[19].regenerated_exclusions records r30 as fixed upstream by PolicyEngine/policyengine-us#10033, and its scenario_100 SNAP recheck gives $6,924 while retaining the unlisted-input exclusion." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The correct answer applies 2026 law under OBBBA, where the TCJA rates, QBI deduction and no personal exemption continue, using single brackets of 10% to $12,400, 12% to $50,400 and 22% above that. Itemized deductions are $24,787: $10,914 of charity after the 0.5%-of-AGI floor, $9,432 of SALT (real estate tax plus about $5,367 of Virginia income tax), and $4,442 of medical. The medical figure counts the $2,434.80 Medicare Part B premium and leaves out the $2,000 of over-the-counter costs. On top of that come the phased-down $4,106 senior deduction and the $4,872 QBI deduction, giving taxable income of $72,807 and tax of $10,729.61. Models split into four error groups: some applied pre-TCJA sunset law with personal exemptions and 15%/25% brackets; some dropped the senior deduction; some took the standard deduction or understated itemized deductions because they missed the Virginia income tax and the Part B premium; and some submitted arbitrary or $0 figures that do not follow from their own work.", + "new": "The frozen reference applies 2026 law under OBBBA, retaining the TCJA rates, QBI deduction and no personal exemption, using single brackets of 10% to $12,400, 12% to $50,400 and 22% on the remaining taxable income. The engine itemizes $24,787: $10,914 of charity after the 0.5%-of-AGI floor, $9,432 of SALT including its $5,367.30 Virginia withholding estimate, and $4,442 of medical including a $2,434.80 Medicare Part B premium it imputes and excluding the $2,000 of over-the-counter costs. Neither state income tax paid or withheld nor Medicare enrollment or a Part B premium is listed in the prompt. The phased-down $4,106 senior deduction and $4,872 QBI deduction leave the reference's taxable income at $72,807 and tax at $10,729.61. Some models applied pre-TCJA sunset law or omitted the senior deduction; others used different readings of the unlisted tax and Medicare inputs, including answers matching the engine's alternatives; some submitted figures unsupported by their own calculations.", + "reason": "The prior case-note amendment, adjudication.json and the scenario_114 entry in reference_exclusions.json distinguish the frozen reference's unlisted withholding and Medicare inputs from household facts and record exclusion for prompt ambiguity." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It estimated Virginia income tax at only $3,524 instead of about $5,367. Its medical deduction ($4,007) counted the $2,000 of over-the-counter costs instead of the $2,434.80 Medicare Part B premium. Together these understated itemized deductions by about $2,278, giving taxable income of $75,085 instead of $72,807.", + "new": "It used its own $3,524 Virginia income tax in SALT, while the frozen reference uses a $5,367.30 withholding estimate. Its $4,007 medical deduction counts the $2,000 of over-the-counter costs, which the engine excludes, instead of the $2,434.80 Part B premium the engine imputes. Neither withholding nor the premium is listed in the prompt. These differences put its itemized deductions about $2,278 below the reference's, giving taxable income of $75,085 versus the reference's $72,807.", + "reason": "The prior claude-fable-5.1 amendment and reference_exclusions.json identify $3,524 as the model's own Virginia tax and the disputed reference deductions as unlisted engine inputs." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "it took the standard deduction even though itemized deductions of $24,787 (including VA income tax and Part B) are larger.", + "new": "it took the standard deduction, which the recorded audit also gives with no state income tax paid or Part B premium; the frozen reference itemizes $24,787 by adding a $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior claude-opus-5 amendment and reference_exclusions.json record that the engine itself takes the standard deduction without the unlisted withholding and Part B inputs." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "Its SALT of $7,618.50 used about $3,555 of Virginia income tax instead of $5,367. Its $4,007 medical deduction counted the $2,000 of over-the-counter costs instead of the $2,434.80 Medicare Part B premium. Itemized deductions came out $2,248 too low, and taxable income came out at $75,055 instead of $72,807.", + "new": "Its SALT of $7,618.50 uses its own Virginia tax of about $3,555, while the frozen reference uses a $5,367.30 withholding estimate. Its $4,007 medical deduction counts the $2,000 of over-the-counter costs the engine excludes, instead of the $2,434.80 Part B premium the engine imputes. Neither withholding nor the premium is listed in the prompt. Itemized deductions are $2,248 below the reference's, and taxable income is $75,055 versus the reference's $72,807.", + "reason": "The prior claude-opus-5.5 amendment and reference_exclusions.json distinguish the model's own Virginia liability from the reference's withholding estimate and imputed premium." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It itemized only $21,188 because it underestimated the Virginia income tax in SALT and left the $2,434.80 Medicare Part B premium out of medical expenses. That left taxable income at $76,407 instead of $72,807.", + "new": "It itemized $21,188, below the frozen reference's $24,787. The reference's SALT uses an unlisted $5,367.30 Virginia withholding estimate, and its medical deduction includes an imputed $2,434.80 Part B premium. The model's taxable income is $76,407 versus the reference's $72,807.", + "reason": "The prior claude-sonnet-5.5 amendment and reference_exclusions.json identify the disputed reference deductions as a withholding estimate and an imputed premium rather than household facts." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "It left out the $4,106 OBBBA senior deduction and limited medical to $2,007 without the $2,434.80 Medicare Part B premium. It also underestimated Virginia income tax in SALT and skipped the 0.5% charitable floor, producing taxable income of $80,631.", + "new": "It omitted the $4,106 OBBBA senior deduction and skipped the 0.5% charitable floor. Its SALT uses its own $3,551.53 Virginia tax, while the frozen reference uses an unlisted $5,367.30 withholding estimate. Its $2,007 medical deduction excludes the $2,434.80 Part B premium the engine imputes despite the prompt listing none. Its taxable income is $80,631.", + "reason": "The prior deepseek-v4-pro amendment identifies the model's own Virginia tax, while reference_exclusions.json identifies the reference's withholding and premium as unlisted inputs." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It also understates itemized deductions by about $2,460 because it left out the Medicare Part B premium and the full ~$5,367 of Virginia income tax.", + "new": "Its implied itemized deductions are about $2,460 below the frozen reference's, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium; its explanation does not give the deduction components.", + "reason": "The prior deepseek-v4.1-flash amendment and reference_exclusions.json make this a comparison with engine-supplied deductions, while prompt.md gives no model deduction components supporting the asserted cause." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "along with Virginia income tax and the Part B premium in itemized deductions. That produced taxable income of about $81,300.", + "new": "and its listed itemized deductions include no state income tax. The frozen reference's deductions add an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium. The model reports taxable income of about $81,300.", + "reason": "The prior gemini-3.1-flash-lite-preview amendment and reference_exclusions.json distinguish the model's listed deductions from the engine's unlisted withholding and premium inputs." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "Its $19,256 of itemized deductions also left out Virginia income tax and the Part B premium.", + "new": "Its $19,256 of itemized deductions are below the frozen reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior gemini-3.5-flash amendment and reference_exclusions.json identify the reference deductions as engine estimates rather than established household payments." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "which overstates deductions by about $6,350 compared with the correct taxable income of $72,807.", + "new": "which implies deductions about $6,350 above the frozen reference's and taxable income below its $72,807; the reference depends on unlisted withholding and Part B inputs.", + "reason": "The prior gemini-3.5-flash-lite amendment and reference_exclusions.json reject presenting the estimate-based frozen computation as the established correct household result." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "Its $22,018 of itemized deductions also missed about $2,770 because it underestimated Virginia income tax and left out the $2,434.80 Medicare Part B premium, giving taxable income of $79,682.", + "new": "Its $22,018 of itemized deductions are about $2,770 below the frozen reference's, whose SALT includes an unlisted $5,367.30 Virginia withholding estimate and whose medical deduction includes an imputed $2,434.80 Part B premium. The model's taxable income is $79,682.", + "reason": "The prior gemini-3.6-flash amendment and reference_exclusions.json require comparison with the engine's estimate-based deductions rather than calling their absence a household deduction error." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "It took the $18,150 standard deduction even though itemized deductions of $24,787 are larger, and left out the $4,106 senior deduction. That gave taxable income of about $83,550 instead of $72,807.", + "new": "It took the $18,150 standard deduction and omitted the $4,106 senior deduction, giving taxable income of about $83,550. The frozen reference's $72,807 uses $24,787 of itemized deductions that include an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior gemini-3.7-flash amendment and reference_exclusions.json distinguish the engine's estimate-based itemized total and taxable income from established household amounts." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "taxable income after all deductions is $72,807", + "new": "the frozen reference's taxable income after deductions including unlisted engine inputs is $72,807", + "reason": "Reference_exclusions.json and adjudication.json classify this calculation as dependent on unlisted inputs, so $72,807 requires attribution to the frozen reference." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "Taxable income after deductions is $72,807, not zero.", + "new": "The frozen reference's taxable income, after deductions including unlisted engine inputs, is $72,807; the listed income also contradicts the model's zero-tax explanation.", + "reason": "Reference_exclusions.json and adjudication.json classify this calculation as dependent on unlisted inputs, so $72,807 requires attribution to the frozen reference." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "Its $17,518 of itemized deductions also left out Virginia income tax and the Part B premium.", + "new": "Its $17,518 itemized total is below the frozen reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior gpt-5.5 amendment and adjudication.json identify the withholding estimate and premium as unlisted engine inputs, while prompt.md gives no model itemized breakdown supporting the asserted omissions." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "took the standard deduction instead of itemizing. Its taxable income of $67,020 is about $5,790 below the correct $72,807.", + "new": "took the standard deduction, while the frozen reference itemizes using an unlisted Virginia withholding estimate and an imputed Part B premium. Its taxable income of $67,020 is about $5,790 below the reference's $72,807.", + "reason": "The prior gpt-5.6-luna amendment and reference_exclusions.json replace claims of a correct household result with comparison to the estimate-based engine calculation." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "Itemized deductions of $24,787 are larger once Virginia income tax and the Medicare Part B premium are counted, so its taxable income of $78,445 is too high.", + "new": "Its $11,970 matches the recorded audit's $11,969.80 standard-deduction result with no state income tax paid, no local sales-tax estimate and no Medicare Part B premium. The frozen reference instead counts a $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium, itemizes $24,787 and reaches taxable income of $72,807.", + "reason": "The prior gpt-5.6-sol amendment and reference_exclusions.json record that its answer matches the engine without the unlisted withholding and Part B inputs, so the standard-deduction choice is not established as a model error." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "Its $11,540 implies taxable income of about $76,490, $3,683 above the correct $72,807. It claimed the age-65 additional standard deduction, which itemizers cannot take, instead of the $4,106 senior deduction, and it understated itemized deductions by missing the Part B premium and the full Virginia income tax.", + "new": "Its $11,540 implies taxable income of about $76,490, about $3,684 above the frozen reference's $72,807. Its explanation gives no deduction figures; if its other deductions match the reference's, its itemized deductions are about $3,684 below the reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior gpt-5.6-terra amendment and prompt.md say the model gave no deduction figures, and adjudication.json identifies the disputed reference deductions as unlisted engine inputs rather than household facts." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "Its $20,639 of itemized deductions is about $4,150 short, because it underestimated the ~$5,367 of Virginia income tax in SALT and left out the $2,434.80 Medicare Part B premium in medical. Taxable income came out at $76,955 instead of $72,807.", + "new": "Its $20,639 of itemized deductions and $11,642.20 answer match the recorded $20,639.28 and $11,642.16 alternative with no Part B premium and the engine's $3,654.15 Virginia liability in SALT. The frozen reference instead counts an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium, producing taxable income of $72,807.", + "reason": "The prior gpt-6-astra amendment and reference_exclusions.json record that this answer matches the no-premium, Virginia-liability alternative rather than an understatement of stated household deductions." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "its roughly $21,000 of itemized deductions missed the Part B premium and part of the Virginia income tax.", + "new": "its implied roughly $21,000 of itemized deductions are below the frozen reference's $24,787, which includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior gpt-6-luna amendment and prompt.md identify the itemized amount as implied, while adjudication.json identifies the disputed reference inputs as unlisted rather than household deductions the model missed." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "instead of the larger $24,787 itemized total.", + "new": "while the frozen reference itemizes $24,787 using an unlisted Virginia withholding estimate and an imputed Medicare Part B premium.", + "reason": "Reference_exclusions.json and adjudication.json identify the inputs making the reference's itemized total larger as unlisted withholding and an imputed Part B premium." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It itemized about $20,640, roughly $4,150 short of $24,787, because it underestimated Virginia income tax in SALT and left the $2,434.80 Medicare Part B premium out of medical expenses. That put taxable income at about $76,955.", + "new": "It gave no itemized figure, and its $11,642.09 answer matches the recorded $11,642.16 alternative with no Part B premium and the engine's $3,654.15 Virginia liability in SALT, where itemized deductions are $20,639.28. The frozen reference instead includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium.", + "reason": "The prior gpt-6.1-sol amendment and prompt.md state that the model gave no itemized figure, while reference_exclusions.json records the matching alternative reading of the unlisted inputs." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "which overstates combined deductions by about $2,250 compared with the correct $33,765", + "new": "which implies combined deductions about $2,250 above the frozen reference's $33,765", + "reason": "The prior grok-4.3 amendment and reference_exclusions.json reject presenting the estimate-based reference deduction total as the established correct household amount." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "Itemized deductions are $24,787 once Virginia income tax ($5,367) and the Medicare Part B premium are included, so its taxable income of $79,445 is too high.", + "new": "The frozen reference itemizes $24,787 by counting an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium. On its own standard-deduction path, the model omitted the $1,000 non-itemizer charitable deduction, leaving taxable income of $79,445.", + "reason": "The prior grok-4.7 amendment distinguishes the unlisted engine inputs from household facts and preserves the separate non-itemizer charitable deduction omission, consistent with reference_exclusions.json." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-build-0.1", + "old": "its SALT used only $2,746 of Virginia income tax instead of about $5,367. Its medical deduction also counted over-the-counter costs instead of the Part B premium, giving taxable income of $79,437.", + "new": "its SALT used its own $2,746 Virginia tax, while the frozen reference uses a $5,367.30 withholding estimate. Its medical deduction counted over-the-counter costs, which the engine excludes, while the reference includes an imputed $2,434.80 Part B premium; neither withholding nor the premium is listed in the prompt. Its taxable income was $79,437.", + "reason": "The prior grok-build-0.1 amendment identifies $2,746 as the model's own Virginia tax, while reference_exclusions.json identifies the disputed reference inputs as engine estimates." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "That total left out the ~$5,367 of Virginia income tax in SALT, the $2,434.80 Medicare Part B premium in medical, and applied no charitable floor; correctly itemized deductions are $24,787, which beats the $19,150 standard-plus-charitable total.", + "new": "The model gives no itemized breakdown. Its $11,969.85 matches the recorded audit's $11,969.80 standard-deduction result with no state income tax paid, no local sales-tax estimate and no Part B premium. The frozen reference instead counts an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium, itemizes $24,787 and returns $10,729.61.", + "reason": "The prior kimi-k3 amendment and reference_exclusions.json record that its answer matches the engine without the unlisted withholding and Part B inputs, so the reference's itemized total cannot be asserted as the correct household result." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "Its taxable income of $78,077 comes from itemized deductions that left Virginia income tax out of SALT and swapped the over-the-counter costs in for the $2,434.80 Medicare Part B premium. That understates deductions by about $5,270.", + "new": "Its taxable income of $78,077 is about $5,270 above the frozen reference's $72,807. The reference's itemized deductions include an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium, and exclude the $2,000 of over-the-counter costs; the model gives no itemized breakdown.", + "reason": "Prompt.md gives no itemized breakdown for muse-wonder, and adjudication.json identifies the withholding estimate and Part B premium as unlisted engine inputs rather than household deductions the model understated." + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "It put itemized deductions at about $19,500 and chose the standard deduction, even though itemized deductions including Virginia income tax and the Part B premium total $24,787.", + "new": "It put itemized deductions at about $19,500 and chose the standard deduction, while the frozen reference itemizes $24,787 by counting an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium.", + "reason": "The prior ox-alpha amendment and reference_exclusions.json identify the disputed deductions as unlisted engine inputs rather than stated household payments." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The filer must itemize $19,419.94:", + "new": "The frozen engine calculation itemizes $19,419.94, counting an unlisted Part B premium:", + "reason": "The scenario_114 state-tax entry in reference_exclusions.json and prior_amendments.json record that removing both unlisted inputs makes the engine take the standard deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "It itemized $19,518, which skips the 0.5%-of-AGI charitable floor, counts the $2,000 OTC spending and leaves out the $2,434.80 Part B premium. It also dropped the $800 aged exemption. Its own steps reached about $3,555, yet it reported an unsupported $2,597 instead of the correct $65,593.83 of taxable income and $3,514.15 of tax.", + "new": "It itemized $19,518, skipping the 0.5%-of-AGI charitable floor and counting $2,000 of OTC spending. The frozen engine instead counts an unlisted $2,434.80 Part B premium and excludes OTC spending, giving $19,419.94 of itemized deductions. It also dropped the $800 aged exemption. Its own steps reached about $3,555, yet it reported an unsupported $2,597.", + "reason": "reference_exclusions.json identifies the frozen state-tax result as dependent on an unlisted Part B premium, so it cannot be described as the unique correct result." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "Its $18,985 of itemized deductions counts the $2,000 OTC spending in medical costs and leaves out the $2,434.80 Part B premium, so it understates itemized deductions by about $435.", + "new": "Its $18,985 of itemized deductions counts the $2,000 OTC spending in medical costs. The frozen engine excludes OTC spending and instead imputes an unlisted $2,434.80 Part B premium, giving an itemized total about $435 higher.", + "reason": "reference_exclusions.json and the prior claude-fable-5.1 amendment identify the $435 difference as the engine's imputed premium less the model's OTC expenses, rather than a required premium the model omitted." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "It never used VA AGI of $86,744 or the Virginia itemized deductions and exemptions, which leave $65,594 taxable.", + "new": "It never used VA AGI of $86,744. The frozen engine's itemized deductions and exemptions leave $65,594 taxable, with the medical deduction counting an unlisted Part B premium.", + "reason": "reference_exclusions.json and the prior claude-haiku-4.5 amendment identify the frozen deduction and taxable-income figures as dependent on an imputed Part B premium." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-5.5", + "old": "so $65,594 stays taxable.", + "new": "so the income is not wiped out; the frozen engine leaves $65,594 taxable after counting an unlisted Part B premium in itemized deductions.", + "reason": "reference_exclusions.json records a different taxable-income result without the unlisted Part B premium, so $65,594 is not an unconditional household outcome." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It took an $8,750 standard deduction instead of the $19,419.94 itemized total and left out the $1,730 of exemptions, which yields $4,227.", + "new": "It took an $8,750 standard deduction and left out the $1,730 of exemptions, which yields $4,227. The frozen engine itemizes $19,419.94 using unlisted inputs; without both its Part B premium and its Virginia withholding estimate in federal SALT, it also takes the $8,750 Virginia standard deduction, yielding $4,127.67 with the exemptions.", + "reason": "reference_exclusions.json and the prior claude-opus-4.7 amendment establish that federal itemization depends on the imputed premium or withholding estimate rather than a stated household fact." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "It used the $8,500 standard deduction instead of itemizing $19,419.94 and turned the exemptions into a vague tax reduction.", + "new": "It used an $8,500 standard deduction and turned the exemptions into a vague tax reduction. The frozen engine itemizes $19,419.94, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, the engine also takes the standard deduction.", + "reason": "The scenario_114 state-tax entry in reference_exclusions.json records that removing both unlisted inputs makes the engine take the standard deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "It itemized $19,518, which skips the 0.5%-of-AGI charitable floor, counts the $2,000 OTC spending and leaves out the $2,434.80 Part B premium. It subtracted only the $930 personal exemption and missed the $800 aged exemption, so it taxed $66,296 instead of $65,593.83.", + "new": "It itemized $19,518, skipping the 0.5%-of-AGI charitable floor and counting $2,000 of OTC spending. The frozen engine instead imputes an unlisted $2,434.80 Part B premium and excludes OTC spending, giving $19,419.94 of itemized deductions. It subtracted only the $930 personal exemption and missed the $800 aged exemption, taxing $66,296 compared with the frozen engine's $65,593.83.", + "reason": "reference_exclusions.json identifies the frozen taxable-income figure as dependent on an unlisted Part B premium rather than a uniquely required correction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It took a $9,000 standard deduction instead of the $19,419.94 itemized total.", + "new": "It took a $9,000 standard deduction. The frozen engine itemizes $19,419.94, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "reference_exclusions.json and the prior claude-sonnet-4.6 amendment establish that the engine's federal itemization depends on the withholding estimate or imputed premium." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "It misstated federal AGI as $104,874 and used a roughly $4,800 standard deduction plus $800 instead of the $19,419.94 itemized total. That left $78,515 taxable instead of $65,594.", + "new": "It misstated federal AGI as $104,874 and used a roughly $4,800 standard deduction plus $800, leaving $78,515 taxable. The frozen engine leaves $65,594 taxable using $19,419.94 of itemized deductions, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "reference_exclusions.json records that the frozen taxable-income figure includes an unlisted premium and that removing both unlisted inputs changes the deduction election." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "It also used the $8,500 standard deduction instead of the $19,419.94 itemized total and left out the $1,730 of exemptions.", + "new": "It also used an $8,500 standard deduction and left out the $1,730 of exemptions. The frozen engine itemizes $19,419.94, including an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "reference_exclusions.json records a standard-deduction reading without both unlisted inputs, while the prior deepseek-v4-pro amendment rejects treating the frozen base as uniquely correct." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "Its $19,518.04 itemized total skips the 0.5%-of-AGI charitable floor, counts the $2,000 OTC spending and leaves out the $2,434.80 Part B premium. These errors partly offset, so it taxed $65,495.96 instead of $65,593.83 and came out about $6 low.", + "new": "Its $19,518.04 itemized total skips the 0.5%-of-AGI charitable floor and counts $2,000 of OTC spending. The frozen engine excludes OTC spending and instead imputes an unlisted $2,434.80 Part B premium. These different assumptions put its taxable income at $65,495.96 compared with the frozen engine's $65,593.83, and its tax about $6 below the frozen reference.", + "reason": "reference_exclusions.json and the prior deepseek-v4-pro-0813 amendment identify the premium as imputed, so omitting it cannot be grouped with the model's errors." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It also left out the Part B premium in itemized deductions and missed the $800 aged exemption, so it taxed $88,125 instead of $65,594.", + "new": "It also missed the $800 aged exemption and taxed $88,125. The frozen engine's $65,594 taxable income includes a medical deduction for an unlisted Part B premium, which the model did not count.", + "reason": "reference_exclusions.json identifies the $65,594 frozen taxable-income figure as dependent on an unlisted Part B premium." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "It overstated itemized deductions at $23,948.60; the correct figure is $19,419.94 with the charitable floor and correct medical amount.", + "new": "Its $23,948.60 itemized total exceeds the frozen engine's $19,419.94, which applies the charitable floor and counts an unlisted Part B premium in the medical deduction.", + "reason": "reference_exclusions.json and the prior gemini-3-flash-preview amendment identify $19,419.94 as the engine's premium-dependent total rather than the uniquely correct deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "Its $4,180 matches about $77,170 of taxable income, which means it took a standard deduction instead of itemizing $19,419.94. The correct path also subtracts $1,730 of exemptions, leaving $65,594.", + "new": "Its $4,180 matches about $77,170 of taxable income, consistent with a standard deduction. The frozen engine itemizes $19,419.94 and subtracts $1,730 of exemptions, leaving $65,594 taxable; without both its unlisted Part B premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "reference_exclusions.json and the prior gemini-3.1-flash-lite-preview amendment reject treating the premium-inclusive deductions as the unique correct path." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "Its $17,518.04 itemized total puts medical costs at $2,007 because it leaves out the $2,434.80 Part B premium; the correct figure is $4,441.86. It also skips the 0.5%-of-AGI charitable floor, so it taxed $67,496 instead of $65,594.", + "new": "Its $17,518.04 itemized total includes a medical deduction of about $2,007, matching the engine's $2,007.06 without the unlisted Part B premium. The frozen engine imputes that $2,434.80 premium and deducts $4,441.86 of medical expenses. The model also skips the 0.5%-of-AGI charitable floor, taxing $67,496 compared with the frozen engine's $65,594.", + "reason": "reference_exclusions.json records $2,007.06 as the medical deduction without the unlisted premium, contradicting the claim that $4,441.86 is uniquely correct." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It itemized $17,518, which leaves out the $2,434.80 Part B premium and skips the charitable floor; the correct total is $19,419.94. It therefore taxed $67,496 instead of $65,593.83.", + "new": "It itemized $17,518 without the Part B premium or the charitable floor. The frozen engine's $19,419.94 itemized total applies the floor and includes an unlisted $2,434.80 premium. The model taxed $67,496 compared with the frozen engine's $65,593.83.", + "reason": "reference_exclusions.json identifies $19,419.94 as dependent on an unlisted Part B premium rather than the unique correct deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "It also used a standard deduction instead of the $19,419.94 itemized total, and the bracket arithmetic is broken.", + "new": "It also used a standard deduction, while the frozen engine itemizes $19,419.94 using an unlisted Part B premium. Without both that premium and its Virginia withholding estimate in federal SALT, the engine also takes the standard deduction. Its bracket arithmetic is broken.", + "reason": "reference_exclusions.json and the prior glm-5.2 amendment establish that using the standard deduction cannot itself be diagnosed as wrong from the frozen engine's itemization." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "It used the $8,500 standard deduction instead of the $19,419.94 itemized total and set the aged exemption at $930 instead of $800.", + "new": "It used an $8,500 standard deduction and set the aged exemption at $930 instead of $800. The frozen engine itemizes $19,419.94 using an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "The scenario_114 state-tax entry in reference_exclusions.json records that removing both unlisted inputs makes the engine take the standard deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "VA AGI after the Social Security subtraction is $86,744, and only $21,150 of itemized deductions and exemptions applies, which leaves $65,594 taxable.", + "new": "VA AGI after the Social Security subtraction is $86,744. The frozen engine subtracts about $21,150 of itemized deductions and exemptions, including an unlisted Part B premium in medical costs, leaving $65,594 taxable.", + "reason": "reference_exclusions.json identifies the frozen deduction total and taxable income as dependent on an unlisted Part B premium rather than unconditional household allowances." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "which leaves $65,594 taxable.", + "new": "with the frozen engine leaving $65,594 taxable after counting an unlisted Part B premium in itemized deductions.", + "reason": "reference_exclusions.json records a different taxable-income result without the unlisted Part B premium, so $65,594 is not an unconditional household outcome." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It took the $8,750 standard deduction instead of itemizing $19,419.94.", + "new": "It took the $8,750 standard deduction. Without both the unlisted $2,434.80 Part B premium and $5,367.30 Virginia withholding estimate in federal SALT, the engine does the same, yielding $76,263.77 taxable income and $4,127.67 tax; its $19,419.94 itemized total applies in the frozen calculation that counts the unlisted inputs.", + "reason": "reference_exclusions.json and the prior gpt-5.6-sol amendment record that the engine also takes the model's standard-deduction path without both unlisted inputs." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The correct path subtracts $19,419.94 of itemized deductions and $1,730 of exemptions.", + "new": "The frozen engine subtracts $19,419.94 of itemized deductions, including an unlisted Part B premium, and $1,730 of exemptions.", + "reason": "reference_exclusions.json identifies the frozen itemized total as dependent on an unlisted Part B premium rather than the unique correct path." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It applied the charitable floor correctly, but its medical deduction is only $2,007 because it leaves out the $2,434.80 Part B premium; the correct figure is $4,441.86. That puts itemized deductions at $16,985 instead of $19,419.94.", + "new": "Its $16,985.18 of itemized deductions matches the engine's $16,985.14 without the unlisted Part B premium. Its $3,654.16 answer matches the adjudicated $3,654.15 alternative within a cent; the frozen $3,514.15 reference includes the $2,434.80 premium the engine imputes.", + "reason": "reference_exclusions.json, adjudication.json and the prior gpt-6-astra amendment record that its answer matches the no-premium alternative rather than misstating deductions." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It overstated total deductions and exemptions by about $2,000 compared with the correct $21,149.94.", + "new": "That implied taxable income is about $2,037 below the frozen engine's $65,593.83. The engine's $21,149.94 total of deductions and exemptions includes an unlisted Part B premium.", + "reason": "reference_exclusions.json and the prior gpt-6-luna amendment reject presenting the frozen premium-inclusive total as uniquely correct." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "It reports about $57,229 taxable, which means it overstated total deductions by about $8,400. The correct total is $19,419.94 of itemized deductions plus $1,730 of exemptions, which leaves $65,594.", + "new": "Its about $57,229 taxable income matches carrying into Virginia the $96,572.80 federal AGI stated in its federal answer, about $10,000 below the engine's: $76,744 of Virginia AGI less about $17,785 of itemized deductions and $1,730 of exemptions. The frozen engine's $19,419.94 itemized total includes an unlisted Part B premium. Virginia's rates on $57,229 give about $3,033, about $10 below its submitted $3,043.16.", + "reason": "The gpt-6-sol entry in prior_amendments.json records a $10,000 AGI understatement, rather than an $8,400 excess deduction, and identifies the engine's medical deduction as including an imputed Part B premium." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "Its itemized total of $16,985.18 leaves out the $2,434.80 Part B premium. That puts medical costs at $2,007 instead of $4,441.86 and overstates taxable income by about $2,435.", + "new": "Its $16,985.18 itemized total matches the engine's $16,985.14 without the unlisted Part B premium. Virginia's rates on its $68,028.82 taxable income give $3,654.16, within a cent of the adjudicated $3,654.15 alternative; its submitted $3,654.66 adds a $0.50 arithmetic slip.", + "reason": "reference_exclusions.json, adjudication.json and the prior gpt-6.1-sol amendment record that its deductions match the no-premium alternative, with a separate $0.50 arithmetic slip." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "VA AGI is $86,744, and only $21,150 of deductions and exemptions applies, which leaves $65,594 taxable.", + "new": "VA AGI is $86,744. The frozen engine subtracts about $21,150 of deductions and exemptions, including an unlisted Part B premium in medical costs, leaving $65,594 taxable.", + "reason": "reference_exclusions.json identifies the frozen deduction total and taxable income as dependent on an unlisted Part B premium rather than unconditional household allowances." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "Its $17,518 itemized total leaves out the $2,434.80 Part B premium and skips the 0.5%-of-AGI charitable floor. It therefore taxed $67,496 instead of $65,593.83.", + "new": "Its $17,518 itemized total omits the Part B premium and skips the 0.5%-of-AGI charitable floor. The frozen engine applies the floor and imputes an unlisted $2,434.80 premium; it leaves $65,593.83 taxable compared with the model's $67,496.", + "reason": "reference_exclusions.json identifies $65,593.83 as the frozen engine's premium-dependent taxable income rather than an unconditional required result." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "Its $19,518 itemized total skips the 0.5%-of-AGI charitable floor, counts the $2,000 OTC spending and leaves out the $2,434.80 Part B premium. These partly offsetting errors put taxable income at $65,496 instead of $65,593.83.", + "new": "Its $19,518 itemized total skips the 0.5%-of-AGI charitable floor and counts $2,000 of OTC spending. The frozen engine excludes OTC spending and instead imputes an unlisted $2,434.80 Part B premium. These different assumptions put taxable income at $65,496 compared with the frozen engine's $65,593.83.", + "reason": "reference_exclusions.json identifies the premium as unlisted, so omitting it cannot be grouped with the model's errors." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "Itemized deductions of $19,419.94 apply instead.", + "new": "The frozen engine itemizes $19,419.94 with its unlisted Part B premium. Without both that premium and its $5,367.30 Virginia withholding estimate in federal SALT, it also takes the standard deduction: $8,750 in Virginia, yielding $76,263.77 taxable income and $4,127.67 tax, about $250 less taxable income than the model's $8,500 deduction gives.", + "reason": "reference_exclusions.json and the prior grok-4.7 amendment establish that the engine also takes the standard deduction without both unlisted inputs." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "It used the $8,500 standard deduction instead of the $19,419.94 itemized total.", + "new": "It used an $8,500 standard deduction. The frozen engine itemizes $19,419.94 with its unlisted Part B premium; without both that premium and its $5,367.30 Virginia withholding estimate in federal SALT, it also takes the standard deduction, $8,750 in Virginia, yielding $76,263.77 taxable income and $4,127.67 tax.", + "reason": "reference_exclusions.json and the prior kimi-k3 amendment establish that the engine also takes the standard deduction without both unlisted inputs." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "and itemized deductions and exemptions still leave $65,594 taxable.", + "new": "and the frozen engine's itemized deductions and exemptions leave $65,594 taxable, counting an unlisted Part B premium in medical costs.", + "reason": "reference_exclusions.json and the prior minimax-m3 amendment identify the frozen deduction and taxable-income figures as engine assumptions rather than unconditional household allowances." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "muse-wonder", + "old": "It took the $8,500 standard deduction instead of itemizing $19,419.94.", + "new": "It took an $8,500 standard deduction. The frozen engine itemizes $19,419.94 using an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "The scenario_114 state-tax entry in reference_exclusions.json records that removing both unlisted inputs makes the engine take the standard deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "It also used a $9,500 standard deduction instead of the $19,419.94 itemized total and left out the $1,730 of exemptions.", + "new": "It also used a $9,500 standard deduction and left out the $1,730 of exemptions. The frozen engine itemizes $19,419.94 using an unlisted Part B premium; without both that premium and its Virginia withholding estimate in federal SALT, it also takes the standard deduction.", + "reason": "The scenario_114 state-tax entry in reference_exclusions.json records that removing both unlisted inputs makes the engine take the standard deduction." + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "used a $9,000 standard deduction instead of itemizing", + "new": "used a $9,000 standard deduction (the engine uses $8,750 when neither its unlisted Part B premium nor its Virginia withholding estimate in federal SALT is counted)", + "reason": "The scenario_114 state-tax entry in reference_exclusions.json records that removing both unlisted inputs makes the engine take the standard deduction." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "In fact a 74-year-old homeowner with $2,800 of household gross income and $1,634 of property tax qualifies, with a computed credit of $768 capped at the $375 maximum for elderly households.", + "new": "In fact this 74-year-old homeowner has federal AGI of $0 and $1,634 of property tax, above 3.5% of that AGI, so the flat table gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income formula corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "It never applied the IT-214 real property tax credit, which this homeowner receives through property taxes paid: 50% of the $1,536 excess over 3.5% of HGI, capped at $375 for age 65+.", + "new": "It never applied the IT-214 real property tax credit: this homeowner is 74 with federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) says the household-gross-income formula is repealed and this 65+ filer receives a flat $375 based on federal AGI of $0." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "The model correctly identified the senior RPTC and its $375 maximum, then substituted an unexplained $63, which is a non-elderly cap from a higher income band. At HGI of $2,800 the formula 0.5 × ($1,634 − 3.5% × $2,800) = $768 exceeds the elderly cap, so the credit is $375.", + "new": "The model identified the senior RPTC and mentioned $375, then substituted an unexplained $63. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives a flat $375, so nothing reduces it to $63.", + "reason": "The adjudication record (adjudication.json, reasoning) affirms a flat $375 based on federal AGI of $0 under the 2025 amendment to N.Y. Tax Law 606(e), and prior_amendments.json corrects the repealed formula and unsupported cap explanations." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "The model inflated household gross income to $12,168 by imputing SSI that is not a listed fact, which pushed the head into a higher income band with a 6% rate and a lower cap ($171). With HGI of only $2,800 Social Security, the 3.5% rate and the $375 elderly cap for the ≤$3,000 band apply.", + "new": "The model used the repealed household-gross-income rule, including estimated SSI, to select a $171 income-band cap. Under the rule in force from 2025, this 74-year-old filer has federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, and prior_amendments.json specifically corrects treating the model's household-gross-income calculation as valid." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "The model applied the $75 maximum, which is the cap for households with no member aged 65 or older, even though the head is 74. It also used a 6% income threshold instead of 3.5%. The elderly cap for HGI ≤ $3,000 is $375.", + "new": "The model used the repealed household-gross-income formula with a 6% threshold and applied a $75 maximum it tied to having no dependents. The head is 74 with federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning) affirms $375 from federal AGI and a flat table under the 2025 amendment to N.Y. Tax Law 606(e), while prior_amendments.json corrects both the repealed formula and the unsupported non-elderly-cap explanation, preserving the model's no-dependents rationale from prompt.md." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "Homeowners qualify through property taxes, and 50% of the $1,536 excess far exceeds the $375 elderly cap, so the credit is $375.", + "new": "Homeowners can qualify through property taxes; with federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old filer receives a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income formula corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "A homeowner aged 65+ with HGI ≤ $3,000 and $1,634 in property taxes receives the $375 maximum.", + "new": "This 74-year-old homeowner has federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, so the flat table gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income eligibility rule corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "The model missed the NY real property tax credit, which is refundable and available to homeowners with HGI ≤ $18,000. Here it equals the $375 elderly cap because 50% of property taxes above 3.5% of the $2,800 HGI comes to $768.", + "new": "The model missed the refundable NY real property tax credit. With federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old homeowner receives a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income limit and formula corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "The NY real property tax credit depends only on property taxes and household gross income, and it pays this senior $375.", + "new": "The NY real property tax credit applies here because this 74-year-old filer has federal AGI of $0 and property taxes above 3.5% of that AGI, giving a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) says the credit uses federal AGI and a flat table, contradicting the stated dependence on household gross income." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "That credit gives this elderly homeowner $375: the computed $768 is capped at the 65+ maximum for HGI ≤ $3,000.", + "new": "That credit gives this 74-year-old homeowner a flat $375 because federal AGI is $0 and property taxes exceed 3.5% of that AGI.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) says the household-gross-income formula is repealed and this filer receives $375 from a federal-AGI flat table, rather than by capping a $768 calculation." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "The model correctly computed 50% × ($1,634 − 3.5% × $2,800) = $768 but applied a nonexistent $1,000 senior maximum. The IT-214 cap for a household with a member aged 65+ and HGI ≤ $3,000 is $375, so the credit is $375.", + "new": "The model used the repealed household-gross-income formula to calculate $768 and compared it with a $1,000 senior maximum that does not apply here. With federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old filer receives a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning) affirms $375 from federal AGI and a flat table under the 2025 amendment to N.Y. Tax Law 606(e), and prior_amendments.json specifically corrects treating the model's $768 calculation as a valid credit calculation." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "With $1,634 in property taxes against a 3.5% threshold on $2,800 HGI, the credit reaches the $375 senior maximum.", + "new": "With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income threshold corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "The real property tax credit counts Social Security in household gross income and pays this senior homeowner the $375 cap.", + "new": "The real property tax credit uses federal AGI and a flat table; with federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old homeowner receives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income basis corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "The model imputed unlisted SSI to reach $12,168 HGI and applied a $75 cap, which is the non-elderly maximum, to a 74-year-old. With HGI of $2,800, the 65+ cap for the ≤$3,000 band is $375.", + "new": "The model used the repealed household-gross-income rule, including estimated SSI, and applied a $75 cap it described as the cap for a homeowner aged 65 or older. The head is 74 with federal AGI of $0 and property taxes above 3.5% of that AGI, so the flat table gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning) affirms $375 from federal AGI and a flat table under the 2025 amendment to N.Y. Tax Law 606(e), while prior_amendments.json corrects the repealed income rule and preserves prompt.md's description of the model's $75 cap as applying to a homeowner aged 65 or older." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The credit is 50% of property taxes above 3.5% of HGI: 0.5 × ($1,634 − $98) = $768, capped at $375 for a 65+ household.", + "new": "From 2025 the credit uses federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income formula corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "The model added estimated SSI to household gross income even though SSI is not a listed fact, which placed the head in a higher income band with a $171 cap. Social Security alone gives HGI of $2,800, which falls in the lowest band with a $375 elderly cap.", + "new": "The model used the repealed household-gross-income rule, including estimated SSI, to select a $171 income-band cap. With federal AGI of $0 and property taxes above 3.5% of that AGI, this 74-year-old filer receives a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income rule corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "The NY real property tax credit applies to this homeowner with HGI ≤ $18,000, and the computed $768 is capped at the $375 maximum for a household with a member aged 65+.", + "new": "The NY real property tax credit applies to this 74-year-old homeowner with federal AGI of $0 and property taxes above 3.5% of that AGI, giving a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income limit and formula corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "It never evaluated the real property tax credit, which pays this homeowner aged 65+ with HGI ≤ $3,000 the $375 cap.", + "new": "It never evaluated the real property tax credit, which gives this 74-year-old homeowner a flat $375 based on federal AGI of $0 and property taxes above 3.5% of that AGI.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) says the household-gross-income rule is repealed and this filer receives $375 from a federal-AGI flat table." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "Age 74 and HGI of $2,800 are exactly what qualify it for the NY real property tax credit at the $375 senior maximum.", + "new": "Age 74, federal AGI of $0, and property taxes above 3.5% of that AGI qualify it for the NY real property tax credit at a flat $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended in 2025) gives $375 from federal AGI and a flat table, whereas this passage repeats the repealed household-gross-income qualification rule corrected in prior_amendments.json." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "case_annotation", + "old": "Household gross income counts the $2,800 of Social Security, which puts the head in the lowest band (HGI ≤ $3,000). In that band the credit is 50% of property tax above 3.5% of HGI: 0.5 × ($1,633.50 − $98) ≈ $768. Because a household member is 65 or older, the credit is capped at $375.", + "new": "This 74-year-old filer has federal AGI of $0 and property taxes of $1,633.50, above 3.5% of that AGI, so the flat table in force from 2025 gives $375.", + "reason": "The adjudication record (adjudication.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) affirms the federal-AGI flat-table rule, and prior_amendments.json specifically corrects presenting the repealed household-gross-income formula and its $768 calculation as governing." } ] } \ No newline at end of file diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf index bc7f7329..8e0ed4f5 100644 Binary files a/app/public/paper/policybench.pdf and b/app/public/paper/policybench.pdf differ diff --git a/app/public/paper/web/figures/positive_zero_scatter.png b/app/public/paper/web/figures/positive_zero_scatter.png index 8bf8d3cd..ba284cc7 100644 Binary files a/app/public/paper/web/figures/positive_zero_scatter.png and b/app/public/paper/web/figures/positive_zero_scatter.png differ diff --git a/app/public/paper/web/index.html b/app/public/paper/web/index.html index 1de6a8b4..ea4f611a 100644 --- a/app/public/paper/web/index.html +++ b/app/public/paper/web/index.html @@ -151,7 +151,7 @@

Table of contents

Abstract

-

PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset and scoring answers against reference outputs from policyengine-us 2.38.6. It evaluates 47 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 86% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 69.9% weighted exact by 25.6 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-10-10), the top-scoring US model on exact match is GPT-6 Sol at 95.6%, while GPT-5.4 nano is lowest at 65.6%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 76.2% exact, below the 80.9% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 8,001 rows whose legacy threshold score is below 1. That universe contains 7,997 of the snapshot’s 7,997 exact-match misses and 4 exact hits; 2,138 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-10-10 with policyengine-us 2.38.6, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 10 outputs whose references rest on engine defects the audits found and upstream has not fixed, the 46 whose references depend on an input the prompt does not state, and two whose references rest on an amount a state published after the freeze. The 0.7% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at https://policybench.org.

+

PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset and scoring answers against reference outputs from policyengine-us 2.41.0. It evaluates 48 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 85% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 69.7% weighted exact by 25.5 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-10-13), the top-scoring US model on exact match is GPT-6 Sol at 95.2%, while GPT-5.4 nano is lowest at 65.4%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 75.9% exact, below the 80.6% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 8,503 rows whose legacy threshold score is below 1. That universe contains 8,499 of the snapshot’s 8,499 exact-match misses and 4 exact hits; 2,218 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-10-13 with policyengine-us 2.41.0, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 47 outputs whose references depend on an input the prompt does not state and the two whose references rest on an amount a state published after the freeze. No excluded output rests on an engine defect the audits found that policyengine-us 2.41.0 still has. The 0.7% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at https://policybench.org.

Introduction

Household tax-and-benefit estimation sits between arithmetic and policy discussion. Each case has numeric labels, but generating those labels requires filing status, household composition, income concepts, program thresholds, and jurisdiction-specific rules. PolicyBench measures whether models can map household records to those outputs.

Most current language-model evaluations do not test this mapping directly. General math benchmarks emphasize symbolic manipulation and exact final answers (Cobbe et al. 2021; Hendrycks et al. 2021), while generic question-answering benchmarks emphasize recall or instruction following. PolicyBench instead asks whether models can transform household facts into policy outputs without access to tools or simulators.

@@ -163,11 +163,11 @@

The nearest public-benefits benchmark is the concurrent Public Benefits Bench, built by Vals AI with the Center for Civic Futures and Code for America (Vals AI 2026). It evaluates models on 459 expert-validated SNAP guidance scenarios (230 in the published test set) spanning all 50 US states plus Guam and the Virgin Islands, and grades free-text answers against expert-written rubric criteria using a judge model that agreed with SNAP policy experts on 80.6% of grading decisions. Where PolicyBench isolates parametric knowledge by withholding tools, Public Benefits Bench varies tool access directly: across models, multi-turn conversation raises pass rates by roughly 7.6 percentage points and web search by about 6.9. The two benchmarks probe complementary failure surfaces — conversational guidance quality for a single program with tools available, versus no-tool numeric estimation across tax and transfer outputs scored deterministically against microsimulation references — and reach consistent conclusions. The best model passes 71.7% of rubric criteria under Public Benefits Bench’s strongest condition, and its authors conclude that “no general-purpose AI model performs well enough to be trusted with SNAP benefits guidance,” matching the deployability gaps PolicyBench reports for unaided calculation.

Finally, PolicyBench depends on two infrastructure literatures that are not themselves large language model (LLM) benchmarks. One is structured-output reliability, since the benchmark relies on multi-output JSON responses and parse coverage rather than free-form prose (Shorten et al. 2024). The other is tax-benefit microsimulation, where systems such as EUROMOD provide the methodological precedent for evaluating policy rules over household microdata (Sutherland and Figari 2013). There is also operational work on applying and evaluating AI systems in public-benefits settings, including caseworker-assist and Supplemental Nutrition Assistance Program (SNAP)-focused evaluations (Nava Labs 2026, 2025; ZenML LLMOps Database 2025). PolicyBench combines these strands into a public cross-model benchmark over household-level tax and benefit outputs.

Benchmark design

-

The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. 10 of the 47 models received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.

+

The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. 10 of the 48 models received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.

Headline metric: exact match

The headline metric is the exact-match rate: the share of requested outputs the model gets right to the dollar (or, for eligibility flags, to the boolean). This is the deployability bar. A tax filer, a benefit estimator, or a caseworker cannot ship “close” — a prediction off by $50 is no more usable than one off by $500, because neither matches the bottom line a downstream system or a household needs. For currency amounts, “exact” means within 1 currency unit of the reference value after numeric parsing. For binary outputs, “exact” means the parsed value is exactly the same 0/1 flag as the reference.

-

Exact match can be a deceptive headline in a zero-inflated panel — but only when the rate is computed unweighted. In the frozen manuscript snapshot, 86% of reference outputs are exact zeros: most sampled households are not eligible for any given program, owe no capital gains tax, do not phase into a refundable credit, and so on. A model that hedges to zero on every requested output earns full exact credit on every zero case, so a naive unweighted exact rate is dominated by those zeros — a hedge-to-zero model scores roughly 86% unweighted, leaving little room above the floor to separate models that actually understand the rules.

-

The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 86% row share to 69.9%. An always-zero baseline therefore scores 69.9% weighted exact rather than the 86% it earns unweighted, and the top model clears that baseline by 25.6 points — a margin comparable to within-1%. Across the 47-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.

+

Exact match can be a deceptive headline in a zero-inflated panel — but only when the rate is computed unweighted. In the frozen manuscript snapshot, 85% of reference outputs are exact zeros: most sampled households are not eligible for any given program, owe no capital gains tax, do not phase into a refundable credit, and so on. A model that hedges to zero on every requested output earns full exact credit on every zero case, so a naive unweighted exact rate is dominated by those zeros — a hedge-to-zero model scores roughly 85% unweighted, leaving little room above the floor to separate models that actually understand the rules.

+

The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 85% row share to 69.7%. An always-zero baseline therefore scores 69.7% weighted exact rather than the 85% it earns unweighted, and the top model clears that baseline by 25.5 points — a margin comparable to within-1%. Across the 48-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.

The exact rate is aggregated to a household score and then to a country score, with the same population-derived output weights that the within-1% companion and the bounded score use. Equal household weight preserves comparability across scenarios and slices. The leaderboard reports US results on this fixed test set.

Near-miss companion: within 1%

The within-1% hit rate is the near-miss-tolerant companion to the headline: the share of requested outputs the model gets within one percent of the PolicyEngine reference value. For currency amounts, “within 1%” means |pred − ref| ≤ 0.01 × |ref| when the reference is nonzero and |pred| ≤ 1 (the same one-currency-unit absolute tolerance used by exact match) when the reference is zero. Binary outputs are requested as integer 0/1 eligibility flags and scored identically to the exact-match metric.

@@ -180,7 +180,7 @@

Secondary me

The benchmark requires each model response to include numeric answers and one explanation per requested output. Both the exact-match rate and the bounded score use only the numeric answers. Explanations are retained for scenario exploration and qualitative error analysis; they should not be interpreted as faithful traces of model reasoning.

Because explanations are required, the canonical task measures policy estimation under a public-facing structured-response contract, not isolated arithmetic accuracy. Prompt fairness is part of the benchmark contract. The current release uses one prompt template per country, with no model-specific tuning. Models receive the same household facts and requested outputs. They receive no web or external tool access. The prompt sets unlisted numeric inputs to 0, unlisted boolean or status facts to false, and household characteristics as constant over the tax-benefit year. Provider-specific differences are limited to structured-output transport, request shape, and the reasoning setup in Table 2. The harness sends no reasoning-control or sampling parameters. It leaves reasoning effort, temperature, and related decoding controls at provider defaults. The frozen GPT-5.5 wave uses provider-default reasoning effort.

The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 0.8 points above that row. Claude Opus 5.5, released September 21, 2026, and Claude Sonnet 5.5, released September 28, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; neither has a sensitivity run. The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.

-

Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). 10 of the 47 models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated.

+

Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). 10 of the 48 models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated.

One row, GLM-5.3-Flash (preview), answered as a cloaked OpenRouter listing called “Ox Alpha” and is labeled a preview. It was publicly callable under the canonical whole-scenario request and forced-tool transport, and its date is its OpenRouter listing date. After the row’s run, Z.ai identified the listing as GLM-5.3-Flash (OpenRouter 2026); the row carries the disclosed name and keeps the preview qualifier, because whether the checkpoint that answered is the one later released under that name has not been stated, and it is distinct from the GLM-5.3 row, which is Z.ai’s larger model called under its own id. Its artifact key stays ox-alpha, the name it was queried under.

Frozen snapshot and open-set status

Leaderboard positions are version-sensitive, so manuscript claims refer to the frozen source-run exports in Table 1 rather than to the live site. The committed source-run exports are the manuscript artifacts. The public site exposes the current prompts, predictions, explanations, and reference outputs for transparency. This makes the public leaderboard open-set: models, model providers, or benchmark users could learn from the released cases before later runs. Protected leaderboard claims would require a separate held-out or rotating set.

@@ -206,11 +206,11 @@

Frozen snapshot an Snapshot date -2026-10-10 +2026-10-13 Model response date -2026-06-12 to 2026-10-08 +2026-06-12 to 2026-10-13 Policy period @@ -222,7 +222,7 @@

Frozen snapshot an Frozen export SHA-256 prefix -US 535a51db14f1 +US 5d4d8b872833 Snapshot manifest @@ -238,11 +238,11 @@

Frozen snapshot an US reference outputs SHA-256 prefix -fdd17a57092d +2d00cf595adf Benchmark spec SHA-256 prefix -ce233f8cbb05 +89067ba731e8 Frozen prompt payload SHA-256 prefix @@ -262,7 +262,7 @@

Frozen snapshot an Reference engine -policyengine-us 2.38.6 +policyengine-us 2.41.0 PolicyEngine.py (recorded for provenance) @@ -278,7 +278,7 @@

Frozen snapshot an Models -47 +48 Output groups @@ -299,12 +299,12 @@

Frozen snapshot an -
+
-Table 2: Model run configuration in the frozen manuscript snapshot. Every model answers through its row’s transport, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (tool_choice set to the answer tool) or a JSON object, the latter where the model card or its family default selects JSON, for most such rows because the provider rejects a forced tool call. Supervised-run fingerprints pin answer contract, request shape, and completion ceiling for 18 rows; tool choice for 17 rows; reasoning setup and timeouts for eight rows. Reasoning setup and timeouts for the other ten fingerprinted rows, and all fields for the other 29 rows, are the harness registry as frozen in the snapshot’s serving-configuration file. Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1’s date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3’s weights, announced at its API launch (Willison 2026), shipped on Hugging Face on July 27, 2026 under a custom license (Tech Times 2026). GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run (OpenRouter 2026), and its Released date is its August 20, 2026 OpenRouter listing date. +Table 2: Model run configuration in the frozen manuscript snapshot. Every model answers through its row’s transport, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (tool_choice set to the answer tool) or a JSON object, the latter where the model card or its family default selects JSON, for most such rows because the provider rejects a forced tool call. Supervised-run fingerprints pin answer contract, request shape, and completion ceiling for 19 rows; tool choice for 18 rows; reasoning setup and timeouts for nine rows. Reasoning setup and timeouts for the other ten fingerprinted rows, and all fields for the other 29 rows, are the harness registry as frozen in the snapshot’s serving-configuration file. Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1’s date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3’s weights, announced at its API launch (Willison 2026), shipped on Hugging Face on July 27, 2026 under a custom license (Tech Times 2026). GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run (OpenRouter 2026), and its Released date is its August 20, 2026 OpenRouter listing date.
@@ -333,7 +333,7 @@

Frozen snapshot an

- + @@ -342,7 +342,7 @@

Frozen snapshot an

- + @@ -351,7 +351,7 @@

Frozen snapshot an

- + @@ -360,7 +360,7 @@

Frozen snapshot an

- + @@ -369,7 +369,7 @@

Frozen snapshot an

- + @@ -378,7 +378,7 @@

Frozen snapshot an

- + @@ -387,7 +387,7 @@

Frozen snapshot an

- + @@ -396,7 +396,7 @@

Frozen snapshot an

- + @@ -405,7 +405,7 @@

Frozen snapshot an

- + @@ -414,7 +414,7 @@

Frozen snapshot an

- + @@ -423,7 +423,7 @@

Frozen snapshot an

- + @@ -432,7 +432,7 @@

Frozen snapshot an

- + @@ -441,7 +441,7 @@

Frozen snapshot an

- + @@ -450,7 +450,7 @@

Frozen snapshot an

- + @@ -459,7 +459,7 @@

Frozen snapshot an

- + @@ -468,7 +468,7 @@

Frozen snapshot an

- + @@ -477,7 +477,7 @@

Frozen snapshot an

- + @@ -486,7 +486,7 @@

Frozen snapshot an

- + @@ -495,7 +495,7 @@

Frozen snapshot an

- + @@ -504,7 +504,7 @@

Frozen snapshot an

- + @@ -513,7 +513,7 @@

Frozen snapshot an

- + @@ -522,7 +522,7 @@

Frozen snapshot an

- + @@ -531,7 +531,7 @@

Frozen snapshot an

- + @@ -540,7 +540,7 @@

Frozen snapshot an

- + @@ -549,7 +549,7 @@

Frozen snapshot an

- + @@ -558,7 +558,7 @@

Frozen snapshot an

- + @@ -567,7 +567,7 @@

Frozen snapshot an

- + @@ -576,7 +576,7 @@

Frozen snapshot an

- + @@ -585,7 +585,7 @@

Frozen snapshot an

- + @@ -594,7 +594,7 @@

Frozen snapshot an

- + @@ -603,7 +603,7 @@

Frozen snapshot an

- + @@ -612,7 +612,7 @@

Frozen snapshot an

- + @@ -621,7 +621,7 @@

Frozen snapshot an

- + @@ -630,7 +630,7 @@

Frozen snapshot an

- + @@ -639,7 +639,7 @@

Frozen snapshot an

- + @@ -648,7 +648,7 @@

Frozen snapshot an

- + @@ -657,7 +657,7 @@

Frozen snapshot an

- + @@ -666,7 +666,7 @@

Frozen snapshot an

- + @@ -675,7 +675,7 @@

Frozen snapshot an

- + @@ -684,7 +684,7 @@

Frozen snapshot an

- + @@ -693,7 +693,7 @@

Frozen snapshot an

- + @@ -702,7 +702,7 @@

Frozen snapshot an

- + @@ -711,7 +711,7 @@

Frozen snapshot an

- + @@ -720,34 +720,43 @@

Frozen snapshot an

- + + + + + + + + + + - + - + - + - + - + @@ -761,9 +770,9 @@

Frozen snapshot an

Data and scenario construction

United States

-

The US benchmark is built from the certified PolicyEngine populace dataset (populace_us_2024) using PolicyEngine US. The dataset contains 160,858 people in 75,112 households, with household, tax-unit, SPM-unit, family, marital-unit, and person records for PolicyEngine calculations. The sampled households are filtered to keep a single-tax-unit, single-family, single-Supplemental Poverty Measure (SPM)-unit structure with at least one adult and a supported filing status. Of the 75,112 households, 63,128 (84.0%) pass the filter and form the eligible draw (this filter-pass rate is coincidentally close to the 86% zero-output share discussed under the benchmark design; the two figures are unrelated). The 16.0% excluded by the filter include multi-tax-unit households (e.g., adult roommates), multi-family households, multi-SPM-unit households, and households whose head reports a filing status outside the supported set. These excluded compositions are exactly the kind of cases where federal/state credit allocations and benefit-unit rules become hardest, so the eligible draw is a tractable subset rather than the full distribution of US households. Prompts include nonzero promptable raw inputs across relevant entities rather than a hand-curated summary, so the models see many of the same facts the simulator receives. Filing status is not stated in the prompt; the reference computation infers it from tax-unit role flags. Models therefore see the same household facts that drive the reference filing-status assignment, but they do not receive that assignment as a label.

+

The US benchmark is built from the certified PolicyEngine populace dataset (populace_us_2024) using PolicyEngine US. The dataset contains 160,858 people in 75,112 households, with household, tax-unit, SPM-unit, family, marital-unit, and person records for PolicyEngine calculations. The sampled households are filtered to keep a single-tax-unit, single-family, single-Supplemental Poverty Measure (SPM)-unit structure with at least one adult and a supported filing status. Of the 75,112 households, 63,128 (84.0%) pass the filter and form the eligible draw (this filter-pass rate is coincidentally close to the 85% zero-output share discussed under the benchmark design; the two figures are unrelated). The 16.0% excluded by the filter include multi-tax-unit households (e.g., adult roommates), multi-family households, multi-SPM-unit households, and households whose head reports a filing status outside the supported set. These excluded compositions are exactly the kind of cases where federal/state credit allocations and benefit-unit rules become hardest, so the eligible draw is a tractable subset rather than the full distribution of US households. Prompts include nonzero promptable raw inputs across relevant entities rather than a hand-curated summary, so the models see many of the same facts the simulator receives. Filing status is not stated in the prompt; the reference computation infers it from tax-unit role flags. Models therefore see the same household facts that drive the reference filing-status assignment, but they do not receive that assignment as a label.

Disability in the household facts

-

The household facts carry one disability fact, while the programs apply several. The certified microdata marks a person as disabled when they report any of the six disability-difficulty items in the Current Population Survey, and the prompt passes that flag through as is disabled: 33 of the 177 people in the benchmark households carry it. The law defines disability program by program, and policyengine-us follows it with a separate input for each determination. SSI requires its own disability criterion and no substantial gainful activity (42 U.S.C. 1382c(a)(3)). SNAP counts a member as disabled through receipt-based routes, among them receiving SSI or Social Security disability benefits and a qualifying veterans’ status (7 U.S.C. 2012(j)); policyengine-us approximates the SSI route with SSI disability status, which does not require receiving SSI. Medicare’s disability route before age 65 requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)), the route policyengine-us models. The tax code uses permanent and total disability for dependents (26 U.S.C. 152), retirement on disability for the credit for the elderly and disabled (26 U.S.C. 22), and incapacity for self-care for the child and dependent care credit (26 U.S.C. 21). The general flag enters none of these determinations; in the reference engine it reaches mainly state provisions, and in SNAP only the work-requirement and student rules. One of those state provisions is California’s Working Disabled Program, a Medi-Cal pathway for people with disabilities who work: the program uses SSI’s definition of disability (42 CFR 435.540(a)), and policyengine-us tests the general flag. No benchmark person carries any of the program-specific inputs. The certified build never sets SSI’s disability criterion, and the months of Social Security disability receipt are integer-typed, which the prompt builder does not expose. The references of a person the prompt calls disabled therefore take the non-disabled path through each of these determinations unless another listed fact establishes it: listed Social Security disability income, for example, makes a person disabled for SNAP. A careful reader of is disabled could take either path, so wherever the other reading moves a reference, the output is excluded from scoring for every model (Section 4.3): SSI and SNAP outputs under SSI’s disability criterion, the SNAP output whose engine defect turns on whether its head is a disabled member, and one California Medicaid output, which the other reading moves once the engine’s Working Disabled Program test reads SSI’s definition. Separately, the prompt lists Social Security disability income but not how long it has been received, so the Medicare eligibility of the five household heads with that income, which turns on 24 months of it, is excluded whether or not the prompt calls them disabled. Where the other reading moves nothing, the output stays scored. The next scenario refresh defines the prompt’s disability fact and states the program-specific determinations as separate facts (PolicyEngine/policybench#165, PolicyEngine/microcosm#876).

+

The household facts carry one disability fact, while the programs apply several. The certified microdata marks a person as disabled when they report any of the six disability-difficulty items in the Current Population Survey, and the prompt passes that flag through as is disabled: 33 of the 177 people in the benchmark households carry it. The law defines disability program by program, and policyengine-us follows it with a separate input for each determination. SSI requires its own disability criterion and no substantial gainful activity (42 U.S.C. 1382c(a)(3)). SNAP counts a member as disabled through receipt-based routes, among them receiving SSI or Social Security disability benefits and a qualifying veterans’ status (7 U.S.C. 2012(j)); policyengine-us 1.755.4, behind the September 22 references, approximated the SSI route with SSI disability status, which does not require receiving SSI; 2.41.0 tests receipt (SSI, Social Security disability benefits and the veterans’ statuses). Medicare’s disability route before age 65 requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)), the route policyengine-us models. The tax code uses permanent and total disability for dependents (26 U.S.C. 152), retirement on disability for the credit for the elderly and disabled (26 U.S.C. 22), and incapacity for self-care for the child and dependent care credit (26 U.S.C. 21). The general flag enters none of these determinations; in the reference engine it reaches mainly state provisions, and in SNAP only the work-requirement and student rules. One of those state provisions is California’s Working Disabled Program, a Medi-Cal pathway for people with disabilities who work: the program uses SSI’s definition of disability (42 CFR 435.540(a)), and policyengine-us tests the general flag. No benchmark person carries any of the program-specific inputs. The certified build never sets SSI’s disability criterion, and the months of Social Security disability receipt are integer-typed, which the prompt builder does not expose. The references of a person the prompt calls disabled therefore take the non-disabled path through each of these determinations unless another listed fact establishes it: listed Social Security disability income, for example, makes a person disabled for SNAP. A careful reader of is disabled could take either path, so wherever the other reading moves a reference, the output is excluded from scoring for every model (Section 4.3): SSI and SNAP outputs under SSI’s disability criterion, the SNAP output whose standard utility allowance turns on whether its head is a disabled member and on its energy subsidy’s source, and one California Medicaid output, which the other reading moves once the engine’s Working Disabled Program test reads SSI’s definition. Separately, the prompt lists Social Security disability income but not how long it has been received, so the Medicare eligibility of the five household heads with that income, which turns on 24 months of it, is excluded whether or not the prompt calls them disabled. Where the other reading moves nothing, the output stays scored. The next scenario refresh defines the prompt’s disability fact and states the program-specific determinations as separate facts (PolicyEngine/policybench#165, PolicyEngine/microcosm#876).

The current US release requests 18 scored output groups spanning federal income tax, refundable credits, payroll and self-employment tax, state and local income tax, Supplemental Nutrition Assistance Program (SNAP), Supplemental Security Income (SSI), Temporary Assistance for Needy Families (TANF), school-meal eligibility, and person-level coverage eligibility for the Special Supplemental Nutrition Program for Women, Infants, and Children (WIC), Medicaid, the Children’s Health Insurance Program (CHIP), Medicare, Head Start, and Early Head Start; several of these categories span two scored groups each (federal and state refundable credits, payroll and self-employment tax, state and local income tax, free and reduced-price school meals), which is how the named categories reconcile to the 18 output groups. The source run also requested the Affordable Care Act (ACA) Premium Tax Credit (PTC), but explanation audits showed that the prompt could be misleading when households lacked plan-specific Marketplace information. We therefore preserve those raw responses but exclude PTC from the canonical scored leaderboard until the prompt contract is revised.

The output scope is intentionally narrower than the full PolicyEngine model. Table 3 summarizes the inclusion rule. The benchmark asks for WIC eligibility rather than a WIC dollar amount; WIC dollar values are used only as impact-weight proxies for coverage flags, not as requested model outputs.

@@ -811,16 +820,17 @@

Disability in the household facts

Reference-output credibility

PolicyBench treats PolicyEngine outputs as benchmark reference outputs, not as administrative records. The reference source is nevertheless stronger than an ad hoc answer key: PolicyEngine is open source, used for household calculators and reform analysis, and externally checked in specific domains. No. 10 Downing Street’s data science team adapted PolicyEngine’s open-source microsimulation model for experimental policy simulation, with validation against external projections before use (Woodruff 2026; Ghenis 2026). In the US, PolicyEngine (2024) reports matching the National Bureau of Economic Research (NBER) TAXSIM-35 model (Feenberg and Coutts 1993) to the cent on the vast majority of cases for the 2021 tax year across hundreds of thousands of tax units per state, with state-specific differences documented in the integration tests. We do not restate that comparison as a single percentage because the published source uses qualitative phrasing rather than a headline accuracy number. PolicyEngine has also signed a memorandum of understanding (Ghenis and Makarchuk 2025) with the Federal Reserve Bank of Atlanta for future validation work against its Policy Rules Database (2026). The Atlanta Fed sources are a caveat rather than evidence of completed validation for this benchmark: they document planned collaboration and the comparison source, not finished checks of the frozen PolicyBench outputs. Taken together, these sources support using PolicyEngine as a transparent reference implementation with partial external validation, but they do not validate every benchmark output.

-

This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered 8,001 rows whose legacy threshold score is below 1. That selected universe contains 7,997 exact-match misses and 4 exact hits; 2,138 additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine’s computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer’s hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot’s scored reference outputs come from the fixed engine, policyengine-us 2.38.6. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. Table 4 summarizes the reviewed discrepancy classes and outcomes.

-

The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged 30 references. Adjudication against primary sources and sandbox runs of the reference engine affirmed 5 of them, traced 8 to inputs the prompt never lists, replaced 5 with a regenerated reference, and confirmed 11 as defects in policyengine-us 1.755.4, the version behind the references then. The judge’s stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause’s rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, 8 had been fixed in policyengine-us after the references were frozen (#8839, #9162, #9301, #9313, #9318, #9363 and #9586): capital gain distributions, New York’s renter cap, the CalEITC’s lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, 15 in all, and policyengine-us 2.38.6 contains all 8 fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin’s 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin’s published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model. The exclusion record holds 14 engine-defect exclusions in 12 households across 8 root causes, and 3 of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). policyengine-us 2.38.6 still has the defect behind 9 of them, across 5 root causes. An output returns to scoring when references built on an engine that fixes its defect replace it, unless it also moves under an input the prompt does not state or its reference rests on a further defect that engine has. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household’s survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.

+

This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered 8,503 rows whose legacy threshold score is below 1. That selected universe contains 8,499 exact-match misses and 4 exact hits; 2,218 additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine’s computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer’s hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot’s scored reference outputs come from the fixed engine, policyengine-us 2.41.0. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. Table 4 summarizes the reviewed discrepancy classes and outcomes.

+

The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. Its flags and those of later waves are settled in the adjudication record. Of the 22 judge-flagged references whose decisions this snapshot keeps, adjudication against primary sources and sandbox runs of the reference engine affirmed 5, traced 8 to inputs the prompt never lists, replaced 5 with a regenerated reference, confirmed 3 as defects in policyengine-us 1.755.4, the version behind the references then, and traced 1 to an amount a state published after the freeze. A later engine upgrade that regenerates a reference drops its decision, so earlier releases kept more of them. The judge’s stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause’s rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, 8 had been fixed in policyengine-us after the references were frozen (#8839, #9162, #9301, #9313, #9318, #9363 and #9586): capital gain distributions, New York’s renter cap, the CalEITC’s lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, 15 in all, and policyengine-us 2.41.0 contains all 8 fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin’s 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix did not, so that part was recorded as its own defect, measured on top of the upstream fix and Wisconsin’s published amounts, and the two outputs it moved were excluded (one of them also for another Wisconsin defect) until policyengine-us fixed it too (#10026); both return to scoring on 2.41.0. Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model. The exclusion record holds 5 engine-defect exclusions in 5 households across 3 root causes, and 2 of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). policyengine-us 2.41.0 no longer has the recorded defect behind any of them. An output returns to scoring when references built on an engine that fixes its defect replace it, unless it also moves under an input the prompt does not state or its reference rests on a further defect that engine has. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household’s survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.

The same wave tested a failure mode no judge flagged. policyengine-us fills a parameter’s values past its last encoded year by projecting it with a price index, so a 2026 reference can rest on a forecast rather than on the amount a government published. Every reference was recomputed with each projected 2026 value held at its last encoded value, and each projected amount behind a moved output was then sourced. Where the government published the 2026 amount before the freeze, the reference takes it: Minnesota’s, Michigan’s and Missouri’s brackets, deductions and exemptions, and Maryland’s withholding allowance. policyengine-us reads the allowance only in its estimate of Maryland income tax withheld, which fills the state income tax part of the federal state and local tax deduction; the one Maryland federal output that estimate moves is excluded (reference_audit/2026-10-05). Wisconsin had published its 2026 brackets and deductions too, but every Wisconsin output they move is excluded for an engine defect, so they enter only those outputs’ corrected values. Where no 2026 amount published before the freeze was found, the reference takes the last amount published: California’s indexed amounts, whose 2026 factor rests on June 2026 prices; the IRS optional sales tax tables, whose 2026 edition had not been found by September 22; Idaho’s zero-rate threshold; Maryland’s return deduction; and SNAP’s FY2027 figures, published on 2026-08-21. Illinois’s exemption and Minnesota’s child credit projections matched the published amounts. These corrections change parameter values and no rule, so the affected references were regenerated with the same engine version rather than excluded: 23 references, most of them SNAP. With the upstream fixes, the September 22 release regenerated 26 references in 24 households, 13 of them SNAP. The reference sidecar records each regenerated value with the fix that produced it, and the fixes, the recomputation of every reference under each, and the verification reports are committed with the snapshot (reference_audit/2026-09-22).

-

The SNAP convention as first built also corrected the engine’s SNAP arithmetic, and an independent review of the snapshot separated those corrections from the parameter holds. They are engine defects, each fixed in policyengine-us after the freeze (#9162 and #9318), so the SNAP references apply the fixes: the engine kept cents in the allotment, where 7 CFR 273.10(e)(2)(ii)(A) requires a whole-dollar allotment; it returned a minimum benefit of $23.84 a month, where 7 CFR 273.10(e)(2)(ii)(C) rounds 8 percent of the one-person maximum to the nearest dollar ($24); it floored net income, which 7 CFR 273.10(e)(1)(ii) requires a state to round to the nearest dollar at each step or to calculate by its TANF procedure, which may keep cents; and it compared income with unrounded income standards. Given the engine’s household size, eligibility and floored net income, the Axiom rules engine’s encoding of 7 U.S.C. 2017(a) reproduces the whole-dollar allotment of all 19 households the first SNAP convention module changed, in each month from January to September (171 of 171 monthly comparisons), and the federal minimum benefit; it did not test New Jersey’s state minimum, and it covers the allotment rounding, not the net income rounding the published references also apply. Recomputing SNAP with net income’s cents kept, rather than rounded to the nearest dollar as the upstream fix does, moves no scored reference by more than the dollar tolerance. One SNAP defect is not fixed upstream: the engine still grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 section 10103 ended in July 2025, and the scored output it moves is excluded. That household’s prompt calls its head disabled but states none of the routes by which 7 U.S.C. 2012(j) defines a disabled member, such as receiving SSI or Social Security disability benefits; read the other way, the allowance would stand, and so would the reference the publication conventions and upstream SNAP fixes give ($3,576, which the output would carry if scored), so the output is excluded on either reading. One more SNAP defect was found after the September 22 release and fixed upstream on 2026-09-24: the engine’s values for the SNAP child support option carried the opposite meaning from the one its formulas read. A state may exclude legally obligated child support paid from gross income (7 CFR 273.9(c)(17)); a state that does not must deduct it from net income (273.9(d)(5)). For 2026 the engine excluded it in 37 jurisdictions that USDA’s SNAP State Options Report lists as deducting it, Michigan among them, and deducted it in 9 of the 14 it lists as excluding it. Recomputed under the fix (policyengine-us #9586), one reference moves: a one-person Michigan household that pays child support stayed within the 200 percent gross limit of Michigan’s broad-based categorical eligibility only because the engine excluded the payments, so the engine granted it the $24 monthly minimum; counting the payments, as Michigan does, leaves it ineligible, and the regenerated reference is $0.

+

The SNAP convention as first built also corrected the engine’s SNAP arithmetic, and an independent review of the snapshot separated those corrections from the parameter holds. They are engine defects, each fixed in policyengine-us after the freeze (#9162 and #9318), so the SNAP references apply the fixes: the engine kept cents in the allotment, where 7 CFR 273.10(e)(2)(ii)(A) requires a whole-dollar allotment; it returned a minimum benefit of $23.84 a month, where 7 CFR 273.10(e)(2)(ii)(C) rounds 8 percent of the one-person maximum to the nearest dollar ($24); it floored net income, which 7 CFR 273.10(e)(1)(ii) requires a state to round to the nearest dollar at each step or to calculate by its TANF procedure, which may keep cents; and it compared income with unrounded income standards. Given the engine’s household size, eligibility and floored net income, the Axiom rules engine’s encoding of 7 U.S.C. 2017(a) reproduces the whole-dollar allotment of all 19 households the first SNAP convention module changed, in each month from January to September (171 of 171 monthly comparisons), and the federal minimum benefit; it did not test New Jersey’s state minimum, and it covers the allotment rounding, not the net income rounding the published references also apply. Recomputing SNAP with net income’s cents kept, rather than rounded to the nearest dollar as the upstream fix does, moves no scored reference by more than the dollar tolerance. One SNAP defect outlasted the references’ first two engine moves: the engine granted the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 section 10103 ended in July 2025, until policyengine-us fixed it (#10033). The scored output it moved was excluded, and it stays excluded on 2.41.0, which gives the record’s corrected value ($3,240), because the stated facts do not decide it, as the 2026-09-29 recheck found. The prompt calls the head disabled but states none of the routes by which 7 U.S.C. 2012(j) defines a disabled member, such as receiving SSI or Social Security disability benefits (with one, the allowance would stand, at $3,576), and it does not say whether the household’s $2,000 energy subsidy is a LIHEAP payment or a State-law payment, which 7 U.S.C. 2014(k)(4) treats differently. One more SNAP defect was found after the September 22 release and fixed upstream on 2026-09-24: the engine’s values for the SNAP child support option carried the opposite meaning from the one its formulas read. A state may exclude legally obligated child support paid from gross income (7 CFR 273.9(c)(17)); a state that does not must deduct it from net income (273.9(d)(5)). For 2026 the engine excluded it in 37 jurisdictions that USDA’s SNAP State Options Report lists as deducting it, Michigan among them, and deducted it in 9 of the 14 it lists as excluding it. Recomputed under the fix (policyengine-us #9586), one reference moves: a one-person Michigan household that pays child support stayed within the 200 percent gross limit of Michigan’s broad-based categorical eligibility only because the engine excluded the payments, so the engine granted it the $24 monthly minimum; counting the payments, as Michigan does, leaves it ineligible, and the regenerated reference is $0.

PolicyBench rebuilt the references on 2026-09-29 with policyengine-us 2.15.17, the newest release when it began sweeping the references that day (uploaded 00:23 UTC), in place of 1.755.4. It computes each scored reference with policyengine_us.Simulation. The snapshot also records policyengine.py 6.1.2 for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside 2.15.17. PolicyBench ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA’s FY2027 figures as published values. policyengine-us 2.15.17 counts Maryland county income tax in its state income tax. PolicyBench’s state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP’s work rules read, so the scenario builder also passes each prompt’s stated usual weekly hours under that input’s name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes 4 scored references (Table 4). Two take law published before the freeze that policyengine-us 1.755.4 did not encode: New Jersey’s child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona’s broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York’s Empire State child credit phase-out. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household’s excluded SNAP output also flagged a scored output the move did not change: the head’s Medicaid eligibility. The head’s income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California’s Working Disabled Program requires SSI’s definition of disability, the unlisted input that already excludes the household’s SNAP. policyengine-us 2.15.17 tests the general disability flag there and gives 1 under either reading; with the program’s test reading SSI’s definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another 2 references move by less than the dollar tolerance. PolicyBench re-reviewed the 19 excluded outputs whose values move on 2.15.17; all stay excluded, each with the value it was decided on. policyengine-us 2.17.0, the newest release when PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gives the same value as 2.15.17 for all 1,984 outputs under the same conventions and adapter. The reference sidecar lists every change, and reference_audit/2026-09-28 holds the recomputation of every output, the investigations and the reviews.

A review of release dashboard-data-20260930 on 2026-10-05 found three more inputs that no prompt states behind scored references; no judge had flagged the scored outputs they move (reference_audit/2026-10-05, reference_audit/2026-10-05-medicare-part-b and reference_audit/2026-10-05-payroll). The first is the state income tax in the federal state and local tax deduction, which takes the state income tax a household paid during the year, or general sales tax instead (26 U.S.C. 164(a)(3) and (b)(5)). No prompt states state income tax withheld or paid. policyengine-us fills that input with state_withheld_income_tax, a formula estimate of withholding on the person’s whole federal adjusted gross income rather than the household’s state liability, and the references take the estimate. Read as the household paying its 2026 state income tax during the year, or as paying none, the federal income tax of five households moves, three of them scored. The second is Medicare enrollment and a Part B premium. policyengine-us treats every Medicare-eligible person as enrolled and adds the modeled Part B premium, $2,434.80 for 2026 where the Medicare Savings Program does not pay it, to medical expenses; no prompt states either. The prompt’s instruction to assume program take-up supports the reference, and its rules that unlisted numbers are zero and that unlisted expenses and health coverage are not to be inferred support the other reading. Without the premium, the outputs of one household move: it itemizes, so its federal income tax moves, and Virginia’s itemized deductions start from the federal ones, so its Virginia income tax moves too. The third is whether the employer deducts the employee share of a state paid-leave or disability premium. The payroll output counts mandatory employee state payroll taxes. In Minnesota, Colorado, Massachusetts and New York the law lets the employer deduct an employee share of such a premium from wages, up to a cap, but does not require it, so the employee pays from $0 to the cap at the employer’s choice. The four payroll references from those states count the cap, and the prompt does not state the employer’s choice; read as the amount the law requires of the employee, each is employee federal payroll tax alone. California’s, Connecticut’s, New Jersey’s and Pennsylvania’s employee contributions are owed by the worker and withheld by the employer, and the review left those references as they were. Each audit recomputed every output on policyengine-us 2.15.17 with the publication conventions, under each reading, and each sweep’s baseline reproduces all 1,928 references that release scored. Beyond the one-dollar tolerance, the sweeps move eight scored outputs, which PolicyBench excluded, and two federal income tax outputs already excluded for other reasons; they move no output that is still scored by any amount. The eight records are computed on policyengine-us 2.15.17, whose values are the references.

On 2026-10-06 PolicyBench ruled to exclude ten more outputs. Eight came from a reference adversary (reference_audit/2026-10-05-reference-adversary). It flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give. A judge that cannot yet see how the engine derived the reference works each flagged output from primary law, and only then reconciles its answer with the derivation. The adversary flagged 61 outputs and ran 52, the other nine belonging to other audits, and it held 45 references. An independent check of its seven other verdicts confirmed four as engine defects: Arizona’s 2026 standard deduction left unindexed (A.R.S. 43-1041(H)), Ohio’s medical deduction counting health insurance premiums only for a Medicare-eligible filer with no employer-paid plan (R.C. 5747.01(A)(10)), Colorado’s 2026 sales tax refund paid without the excess state revenues that C.R.S. 39-22-2003(2) requires, and New York’s 2026 child and dependent care credit, which Tax Law 606(c-2) replaced. The check refuted two, the Medicaid eligibility of two children in a North Carolina household, whose references stand. The seventh is a question of definition. A Pennsylvania household’s dependent earns $45,000 of wages and must file their own return, and the output definitions name no return, so the household’s federal and state income tax can be read with or without that return; a scan for the same question found a Missouri household, and PolicyBench excluded the four outputs. The other two outputs are two Louisiana households’ 2026 state income tax (reference_audit/2026-10-05-louisiana). Their references rest on a standard deduction that policyengine-us computed from published price indexes under Louisiana’s formula; Louisiana published the return amount itself on 2026-09-28, after the freeze. All ten records are computed on policyengine-us 2.15.17.

-

On 2026-10-10 PolicyBench moved the references again, from policyengine-us 2.15.17 to 2.38.6, a release that fixes engine defects behind excluded outputs. With the same publication conventions and adapter, it recomputed every output (reference_audit/2026-10-09-engine-upgrade). policyengine-us 2.38.6 was the newest release when PolicyBench began sweeping the references on 2026-10-10 (uploaded 05:39 UTC). It was still the newest release when PolicyBench checked PyPI on 2026-10-10 at 06:53 UTC. The move fixes the engine defects behind 18 outputs. Earlier releases excluded 14 of them, which return to scoring at the new version’s values; the other four, ruled on 2026-10-06 while still scored, stay scored at those values. Of the 18, 16 land within $1 of the corrected values their exclusion records carry. The other two are held instead to the values the audit’s fix modules give on policyengine-us 2.37.2, an engine that still has the defects; each lands within $1 of its own. The reference sidecar records each target with the upstream fix it names (regenerated_exclusions). The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The move changes one scored references beyond the dollar tolerance and 2 within it, and removes two outputs from scoring. The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax). PolicyBench re-reviewed the 17 excluded outputs whose values move on 2.38.6; all stay excluded, each with the value it was decided on. The 58 excluded outputs now keep the values they were decided on (38 computed with policyengine-us 1.755.4, 18 with 2.15.17, 2 with 2.38.6).

+

On 2026-10-10 PolicyBench moved the references from policyengine-us 2.15.17 to 2.38.6, a release that fixes engine defects behind excluded outputs, and recomputed every output under the same publication conventions and adapter (reference_audit/2026-10-09-engine-upgrade). The move fixes the engine defects behind 18 outputs. Earlier releases excluded 14 of them, which return to scoring at the new version’s values; the other four, ruled on 2026-10-06 while still scored, stay scored at those values. Of the 18, 16 land within $1 of the corrected values their exclusion records carry. The other two are held instead to the values the audit’s fix modules give on policyengine-us 2.37.2, an engine that still has the defects; each lands within $1 of its own. The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The move changes one scored reference beyond the dollar tolerance and 2 within it, and removes two outputs from scoring. The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax).

+

On 2026-10-13 PolicyBench moved the references again, from policyengine-us 2.38.6 to 2.41.0, the first release with every fix the previous move was waiting on. With the same publication conventions and adapter, it recomputed every output (reference_audit/2026-10-12-engine-upgrade). policyengine-us 2.41.0 was the newest release when PolicyBench began sweeping the references on 2026-10-13 (uploaded 01:44 UTC). policyengine-us 2.41.2, the newest release when PolicyBench checked PyPI on 2026-10-13 at 11:34 UTC, gives the same value as 2.41.0 for all 1,984 outputs under the same conventions and adapter. The move fixes the engine defects behind nine outputs. Earlier releases excluded them, and they return to scoring at the new version’s values. Of the nine, six land within $1 of the corrected values their exclusion records carry. The other three are held instead to the values the audit’s fix modules give on policyengine-us 2.38.6, an engine that still has the defects; each lands within $1 of its own. The reference sidecar records each target with the upstream fix it names (regenerated_exclusions). The restored outputs take the upstream fixes for Idaho’s subtraction for health insurance premiums (policyengine-us #10023; two outputs), Wisconsin’s retirement income exclusion before refundable credits and Wisconsin’s capital gain subtraction for distributions (policyengine-us #10026; two outputs), the IRA deduction’s active-participant phase-out, California’s itemized deduction conformity and California’s educator expense addback (policyengine-us #10032, #10034, #10052) and elective deferrals in the earned income behind the EITC and the refundable child tax credit (policyengine-us #10030; four outputs). The move changes no scored reference, by any amount, and removes no output from scoring. PolicyBench re-reviewed the 12 excluded outputs whose values move on 2.41.0; all stay excluded, each with the value it was decided on. The 49 excluded outputs now keep the values they were decided on (29 computed with policyengine-us 1.755.4, 18 with 2.15.17, 2 with 2.38.6).

The audit tooling itself required hardening, which we report because it changes how much the annotations can be trusted. The derivation narratives displayed alongside reference values are generated from engine computation traces. After earlier model-written narratives were found to misattribute eligibility pathways, mechanical validators now enforce that each narrative states the engine’s actual mechanism — for example, that a child ineligible for CHIP because they qualify for Medicaid is never described as failing a CHIP income test. The failure-audit judge reads these grounded narratives together with engine facts and must return a per-model diagnosis naming the specific rule or computation the model missed. It is barred from re-adjudicating the reference in prose: hedged or reference-adjudicating verdicts are mechanically rejected and re-judged, and genuine reference doubt is routed to a structured flag that triggers the manual adjudication described above.

-

After freezing the snapshot and completing response-contract repairs, we annotated the 8,001 rows whose legacy threshold score is below 1 and their scenario-output cases. Table 5 reports the final classifications for that legacy-threshold audit universe. It contains 7,997 of the snapshot’s 7,997 exact-match misses and 4 exact hits. The 2,138 other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: llm_error (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or parse_contract_failure (a missing or unparsable answer, classified deterministically). Three further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. prompt_ambiguity marks 42 outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine’s Working Disabled Program test read SSI’s definition, as the program does). reference_engine_defect marks the 14 outputs where that engine misapplies the law on stated facts. policyengine-us 2.38.6 computes four of them at their corrected values; they stay excluded because each also moves under an input the prompt does not state. policyengine-us 2.38.6 computes one more at its recorded corrected value, but its reference rests on a further engine defect, found on 2.38.6 and not fixed there, so it stays excluded. reference_later_law marks the two Louisiana outputs whose reference rests on an amount the state published after the freeze. Together they remove 58 outputs in 42 households, and of the 2,212 annotated rows on those outputs, 2,164 carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP’s October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on 1,926 of its 1,984 requested outputs. The exclusion record (reference_exclusions.json) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records 70 cases of developer adjudication, kept beside the annotations with the judge’s verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the 94 cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29, September 30 and October 9 additions joined or a reference revision changed (314 of 677 cases), its own row’s cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (meets_ssi_disability_criteria), false for every person in the certified June 2026 microdata build; months of SSDI receipt (months_receiving_social_security_disability), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (weekly_hours_worked_before_lsr), which policyengine-us 1.755.4 defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin’s homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers’ child, which decides whose income counts in that dependent’s Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; the state income tax a household withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part and which policyengine-us estimates from federal adjusted gross income; Medicare enrollment and a Part B premium, which policyengine-us assumes for every Medicare-eligible person and counts as a medical expense; whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general is disabled fact and never a program-specific determination (Section 4.2). Every exclusion but the California Medicaid output and the 2026-10-06 records was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect’s sandbox fix, moved on policyengine-us 1.755.4 or, for the state and local tax refund reading and the readings of the 2026-10-05 review, on 2.15.17; the outputs the 2026-10-10 upgrade excluded are ones policyengine-us 2.38.6 itself moved, onto an input no prompt states. The 2026-10-06 records are the exception by design. The reference adversary selects the outputs where models agree against the reference, and an independent check against primary law then decided each; the two Louisiana records came from a sweep of that state’s 2026 standard deduction. On 1.755.4, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On 2.15.17, PolicyBench re-ran four of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves three previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same 2.15.17 calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and three scored outputs move by less. PolicyBench did not re-run the other sweeps on 2.15.17. On 2.15.17, the California Medicaid output stays at 1 under either reading of SSI’s disability criterion, because the engine’s Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household’s excluded SNAP output, and excluded it because the program itself requires SSI’s definition of disability.

+

After freezing the snapshot and completing response-contract repairs, we annotated the 8,503 rows whose legacy threshold score is below 1 and their scenario-output cases. Table 5 reports the final classifications for that legacy-threshold audit universe. It contains 8,499 of the snapshot’s 8,499 exact-match misses and 4 exact hits. The 2,218 other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: llm_error (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or parse_contract_failure (a missing or unparsable answer, classified deterministically). Three further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. prompt_ambiguity marks 42 outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine’s Working Disabled Program test read SSI’s definition, as the program does). reference_engine_defect marks the 5 outputs where that engine misapplies the law on stated facts. policyengine-us 2.41.0 computes all five at their corrected values; they stay excluded because each also moves under an input the prompt does not state. reference_later_law marks the two Louisiana outputs whose reference rests on an amount the state published after the freeze. Together they remove 49 outputs in 37 households, and of the 1,843 annotated rows on those outputs, 1,805 carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP’s October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on 1,935 of its 1,984 requested outputs. The exclusion record (reference_exclusions.json) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records 61 cases of developer adjudication, kept beside the annotations with the judge’s verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the 93 cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29, September 30, October 9 and October 13 additions joined or a reference revision changed (315 of 677 cases), its own row’s cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (meets_ssi_disability_criteria), false for every person in the certified June 2026 microdata build; months of SSDI receipt (months_receiving_social_security_disability), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (weekly_hours_worked_before_lsr), which policyengine-us 1.755.4 defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin’s homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers’ child, which decides whose income counts in that dependent’s Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; the state income tax a household withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part and which policyengine-us estimates from federal adjusted gross income; Medicare enrollment and a Part B premium, which policyengine-us assumes for every Medicare-eligible person and counts as a medical expense; whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general is disabled fact and never a program-specific determination (Section 4.2). Every exclusion but the California Medicaid output and the 2026-10-06 records was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect’s sandbox fix, moved on policyengine-us 1.755.4 or, for the state and local tax refund reading and the readings of the 2026-10-05 review, on 2.15.17; the outputs the 2026-10-10 upgrade excluded are ones policyengine-us 2.38.6 itself moved, onto an input no prompt states. The 2026-10-06 records are the exception by design. The reference adversary selects the outputs where models agree against the reference, and an independent check against primary law then decided each; the two Louisiana records came from a sweep of that state’s 2026 standard deduction. On 1.755.4, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On 2.15.17, PolicyBench re-ran four of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves three previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same 2.15.17 calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and three scored outputs move by less. PolicyBench did not re-run the other sweeps on 2.15.17. On 2.15.17, the California Medicaid output stays at 1 under either reading of SSI’s disability criterion, because the engine’s Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household’s excluded SNAP output, and excluded it because the program itself requires SSI’s definition of disability.

@@ -890,10 +900,14 @@

Reference-output credibility

- - + + + + + + @@ -936,18 +950,18 @@

Reference-output credibility

- - - + + + - - - + + + @@ -961,7 +975,7 @@

Reference-output credibility

Results

United States leaderboard

-

The US leaderboard for the frozen manuscript snapshot is shown in Table 6. The top three models in that snapshot are GPT-6 Sol (95.6 exact, 95% CI 93.9–97.0; 96.2 within-1%), Claude Opus 5.5 (94.4 exact, 95% CI 92.5–96.1; 95.8 within-1%), and GPT-5.6 Sol (93.5 exact, 95% CI 91.0–95.6; 94.4 within-1%).

+

The US leaderboard for the frozen manuscript snapshot is shown in Table 6. The top three models in that snapshot are GPT-6 Sol (95.2 exact, 95% CI 93.5–96.7; 95.8 within-1%), Claude Opus 5.5 (94.1 exact, 95% CI 92.2–95.9; 95.6 within-1%), and GPT-5.6 Sol (93.1 exact, 95% CI 90.6–95.3; 94.0 within-1%).

@@ -1002,52 +1016,52 @@

United States leaderboard

- - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - - + + + - - - - - - - + + + + + + +
Claude Fable 5 claude-fable-5 2026-06-091,926/​1,9261,935/​1,935 forced tool 1 output/​request thinking does not engage under forced tool callClaude Fable 5.1 claude-fable-5-1 2026-09-011,926/​1,9261,935/​1,935 JSON whole scenario provider default; 16,384-token shared budgetClaude Haiku 4.5 claude-haiku-4-5-20251001 2025-10-151,926/​1,9261,935/​1,935 forced tool 1 output/​request provider defaultClaude Haiku 5.5 claude-haiku-5-5 2026-10-071,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetClaude Opus 4.7 claude-opus-4-7 2026-04-161,926/​1,9261,935/​1,935 forced tool 1 output/​request provider defaultClaude Opus 4.8 claude-opus-4-8 2026-05-281,926/​1,9261,935/​1,935 forced tool 1 output/​request provider defaultClaude Opus 5 claude-opus-5 2026-07-241,926/​1,9261,935/​1,935 forced tool whole scenario thinking does not engage under forced tool callClaude Opus 5.5 claude-opus-5-5 2026-09-211,926/​1,9261,935/​1,935 JSON whole scenario provider default; 16,384-token shared budgetClaude Sonnet 4.6 claude-sonnet-4-6 2026-02-171,926/​1,9261,935/​1,935 forced tool 1 output/​request provider defaultClaude Sonnet 5 claude-sonnet-5 2026-06-301,926/​1,9261,935/​1,935 forced tool 1 output/​request thinking does not engage under forced tool callClaude Sonnet 5.5 claude-sonnet-5-5 2026-09-281,926/​1,9261,935/​1,935 JSON whole scenario provider default; 16,384-token shared budgetDeepSeek V4 Flash 0731 openrouter/​deepseek/​deepseek-v4-flash-0731 2026-07-31†1,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 98,304-token shared budgetDeepSeek V4 Pro deepseek/​deepseek-v4-pro 2026-04-24†1,926/​1,9261,935/​1,935 JSON whole scenario provider default; 16,384-token shared budgetDeepSeek V4 Pro 0813 openrouter/​deepseek/​deepseek-v4-pro-0813 2026-08-13†1,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 98,304-token shared budgetDeepSeek V4.1 Flash deepseek/​deepseek-flash 2026-09-10†1,926/​1,9261,935/​1,935 JSON whole scenario provider default; 16,384-token shared budgetGemini 3 Flash Preview gemini/​gemini-3-flash-preview 2025-12-171,926/​1,9261,935/​1,935 JSON whole scenario provider defaultGemini 3.1 Flash Lite Preview gemini/​gemini-3.1-flash-lite-preview 2026-03-031,926/​1,9261,935/​1,935 JSON whole scenario provider defaultGemini 3.1 Pro Preview gemini/​gemini-3.1-pro-preview 2026-02-191,926/​1,9261,935/​1,935 JSON whole scenario provider defaultGemini 3.5 Flash gemini/​gemini-3.5-flash 2026-05-191,926/​1,9261,935/​1,935 JSON whole scenario provider defaultGemini 3.5 Flash-Lite gemini/​gemini-3.5-flash-lite 2026-07-211,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGemini 3.6 Flash gemini/​gemini-3.6-flash 2026-07-211,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGemini 3.7 Flash gemini/​gemini-3.7-flash 2026-08-131,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGemini 3.8 Flash gemini/​gemini-3.8-flash 2026-09-021,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGLM-5.2 openrouter/​z-ai/​glm-5.2 2026-06-13†1,790/​1,9261,799/​1,935 JSON 3 outputs/​request provider default; 16,384-token shared budgetGLM-5.3 openrouter/​z-ai/​glm-5.3 2026-08-14†1,853/​1,9261,860/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-5.4 mini gpt-5.4-mini 2026-03-171,926/​1,9261,935/​1,935 forced tool whole scenario provider defaultGPT-5.4 nano gpt-5.4-nano 2026-03-171,926/​1,9261,935/​1,935 forced tool whole scenario provider defaultGPT-5.5 gpt-5.5 2026-04-231,926/​1,9261,935/​1,935 forced tool 3 outputs/​request provider default; 16,384-token shared budgetGPT-5.6 Luna gpt-5.6-luna 2026-07-091,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-5.6 Sol gpt-5.6-sol 2026-07-091,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-5.6 Terra gpt-5.6-terra 2026-07-091,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-6 Astra gpt-6-astra 2026-09-041,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-6 Luna gpt-6-luna 2026-09-221,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-6 Sol gpt-6-sol 2026-09-221,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGPT-6.1 Sol gpt-6.1-sol 2026-09-291,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGrok 4.3 xai/​grok-4.3 2026-04-171,926/​1,9261,935/​1,935 forced tool whole scenario provider defaultGrok 4.5 xai/​grok-4.5 2026-07-081,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGrok 4.6 xai/​grok-4.6 2026-08-121,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGrok 4.7 xai/​grok-4.7 2026-09-211,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budgetGrok Build 0.1 xai/​grok-build-0.1 2026-05-291,926/​1,9261,935/​1,935 forced tool whole scenario provider defaultInkling openrouter/​thinkingmachines/​inkling 2026-07-15†1,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 49,152-token shared budgetKimi K2.6 openrouter/​moonshotai/​kimi-k2.6 2026-04-20†1,536/​1,9261,538/​1,935 JSON 3 outputs/​request provider default; 16,384-token shared budgetKimi K3 openrouter/​moonshotai/​kimi-k3 2026-07-16†1,866/​1,9261,874/​1,935 JSON whole scenario provider default; 49,152-token shared budgetMiniMax M3 openrouter/​minimax/​minimax-m3 2026-06-01†1,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budget
Muse Wonderopenrouter/​meta/​muse-wonder2026-10-121,935/​1,935JSONwhole scenarioprovider default; 16,384-token shared budget
GLM-5.3-Flash (preview) openrouter/​stealth/​ox-alpha 2026-08-201,926/​1,9261,935/​1,935 forced tool whole scenario provider default; 16,384-token shared budget
Qwen3.7-max openrouter/​qwen/​qwen3.7-max 2026-05-191,926/​1,9261,935/​1,935 JSON 3 outputs/​request provider default; 16,384-token shared budget
Qwen3.8-Max openrouter/​qwen/​qwen3.8-max 2026-08-031,926/​1,9261,935/​1,935 JSON whole scenario provider default; 98,304-token shared budget Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us 1.755.4 and 2.15.17 to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes 4 scored references, for law published before the freeze that 1.755.4 did not encode or misapplied: New Jersey’s 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona’s 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York’s Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: 2.15.17 counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the 19 excluded outputs that move; all stay excluded. The re-review of a California household’s excluded SNAP output flagged its head’s Medicaid eligibility, which turns on the same unlisted input, SSI’s disability criterion, through the Working Disabled Program; PolicyBench excluded it.
US references on policyengine-us 2.38.6 (October 2026 upgrade)The move from policyengine-us 2.15.17 fixes the engine defects behind 18 outputs (14 that earlier releases excluded return to scoring at the new version’s values; the 4 ruled on 2026-10-06 while still scored stay scored) and changes 1 scored reference beyond the dollar tolerance; 2 outputs leave scoring. PolicyBench re-reviewed the 17 excluded outputs whose values move; all stay excluded. The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax).US references on policyengine-us 2.38.6 (October 10 upgrade)The move from policyengine-us 2.15.17 fixes the engine defects behind 18 outputs (14 that earlier releases excluded return to scoring at the new version’s values; the 4 ruled on 2026-10-06 while still scored stay scored). The move changes one scored reference beyond the dollar tolerance and 2 within it, and removes two outputs from scoring. PolicyBench re-reviewed the 17 excluded outputs whose values move; all stay excluded. The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax).
US references on policyengine-us 2.41.0 (October 13 upgrade)The move from policyengine-us 2.38.6 fixes the engine defects behind 9 outputs (earlier releases excluded them, and they return to scoring at the new version’s values). The move changes no scored reference, by any amount, and removes no output from scoring. PolicyBench re-reviewed the 12 excluded outputs whose values move; all stay excluded. The restored outputs take the upstream fixes for Idaho’s subtraction for health insurance premiums (policyengine-us #10023; two outputs), Wisconsin’s retirement income exclusion before refundable credits and Wisconsin’s capital gain subtraction for distributions (policyengine-us #10026; two outputs), the IRA deduction’s active-participant phase-out, California’s itemized deduction conformity and California’s educator expense addback (policyengine-us #10032, #10034, #10052) and elective deferrals in the earned income behind the EITC and the refundable child tax credit (policyengine-us #10030; four outputs).
US federal and Virginia income tax (October 2026 review) No judge flag: a review of release dashboard-data-20260930 swept every reference on policyengine-us 2.15.17 under readings of two inputs no prompt states. The state income tax part of the federal state and local tax deduction is policyengine-us’s formula estimate of state income tax withheld on federal adjusted gross income; read as the household paying its 2026 liability during the year, or paying none, three scored federal income tax outputs move, and PolicyBench excluded them; the readings also move two federal outputs already excluded. policyengine-us treats every Medicare-eligible person as enrolled and counts a modeled Part B premium as a medical expense; without it, one household’s federal and Virginia income tax move, and both are excluded.
US8001734265985037834669 0 0 0
Total8001734265985037834669 0 0 0 0 GPT-6 Sol95.693.9–97.095.293.5–96.7 1–296.297.695.897.3 98.2
1 Claude Opus 5.594.492.5–96.194.192.2–95.9 1–395.897.395.697.2 98.0
2 GPT-5.6 Sol93.591.0–95.693.190.6–95.3 2–794.496.197.294.095.897.1
3 Claude Sonnet 5.592.289.8–94.391.789.4–93.9 3–893.496.096.992.995.796.7
4GPT-6 Luna92.089.8–94.03–993.996.897.5GPT-6 Astra91.789.3–94.03–1092.394.396.9
@@ -1060,9 +1074,9 @@

United States leaderboard

Uncertainty

Each leaderboard score is a mean over a sample of 100 households, so the rankings carry sampling uncertainty. We quantify it with a household-level bootstrap: the 100 households are resampled with replacement 10,000 times, and each model’s population-weighted exact-match rate is recomputed on every resample. The resampling unit is the household, carried with its full output vector, so within-household correlations across outputs are preserved. The same resampled set is used for every model in each replicate (a paired bootstrap), so the rank ranges in Table 6 reflect genuine ordering uncertainty rather than independent per-model noise.

-

Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 6 of the other 46 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.

+

Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 6 of the other 47 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.

Cost and latency

-

Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.001 (Claude Haiku 5.5) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.6% exact, sits mid-pack at $0.027 per household and 23 s.

+

Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.001 (Claude Haiku 5.5) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.2% exact, sits mid-pack at $0.027 per household and 23 s.

@@ -1083,283 +1097,289 @@

Cost and latency

GPT-6 Sol -95.6 +95.2 $0.027 23 s Claude Opus 5.5 -94.4 +94.1 $0.067 24 s GPT-5.6 Sol -93.5 +93.1 $0.090 36 s Claude Sonnet 5.5 -92.2 +91.7 $0.034 20 s -GPT-6 Luna -92.0 -$0.002 -29 s - - GPT-6 Astra -91.9 +91.7 $0.126 32 s + +GPT-6 Luna +91.6 +$0.002 +29 s + Claude Fable 5.1 -91.3 +90.9 $0.257 49 s Kimi K3 -91.0 +90.6 $0.470 1023 s GPT-6.1 Sol -90.3 +90.1 $0.023 34 s +Muse Wonder +90.1 +$0.079 +26 s + + GPT-5.6 Luna -89.2 +88.8 $0.019 14 s - + GLM-5.3-Flash (preview) -89.0 +88.6 — 242 s - + GPT-5.5 -88.2 +87.9 $0.262 — - + Inkling -88.2 +87.8 $0.091 250 s - + Grok 4.7 -88.1 +87.7 $0.260 547 s - + GPT-5.6 Terra -88.0 +87.7 $0.039 19 s - + DeepSeek V4.1 Flash -86.8 +86.4 $0.024 85 s - + Grok 4.6 -86.2 +85.9 $0.087 219 s - + Gemini 3.8 Flash -86.0 +85.6 $0.068 56 s - + Grok 4.5 -85.1 +84.7 $0.073 112 s - + Claude Opus 5 -84.0 +83.7 $0.067 16 s - + Claude Fable 5 -83.7 +83.3 $0.541 98 s - + DeepSeek V4 Pro 0813 -83.4 +83.1 $0.199 808 s - + DeepSeek V4 Flash 0731 -83.3 +83.0 $0.003 195 s - + Gemini 3.7 Flash -83.1 +82.8 $0.014 9 s - + Gemini 3.6 Flash -83.0 +82.7 $0.055 26 s - + Gemini 3.5 Flash-Lite -82.0 +81.6 $0.003 3 s - + Gemini 3.1 Pro Preview -81.6 +81.2 $0.085 45 s - + GLM-5.3 -81.4 +81.1 $0.169 134 s - + Gemini 3 Flash Preview -81.0 +80.7 $0.048 58 s - + Claude Opus 4.7 -80.9 +80.6 $0.293 53 s - + Grok 4.3 -80.7 +80.4 $0.015 22 s - + Claude Sonnet 4.6 -80.3 +80.0 $0.186 114 s - + Grok Build 0.1 -80.2 +79.9 $0.045 135 s - + Gemini 3.5 Flash -80.1 +79.8 $0.056 24 s - + Gemini 3.1 Flash Lite Preview -79.7 +79.4 $0.002 3 s - + DeepSeek V4 Pro -79.7 +79.4 $0.011 170 s - + Claude Haiku 5.5 -79.6 +79.3 $0.001 5 s - + Qwen3.7-max -77.4 +77.0 $0.139 — - + GLM-5.2 -76.9 +76.6 $0.082 — - + Claude Opus 4.8 -76.2 +75.9 $0.255 55 s - + MiniMax M3 -75.8 +75.5 $0.007 — - + Qwen3.8-Max -75.2 +74.8 $0.224 861 s - + Claude Haiku 4.5 -74.8 +74.6 $0.055 48 s - + GPT-5.4 mini -73.6 +73.3 $0.006 5 s - + Claude Sonnet 5 -72.7 +72.4 $0.172 64 s - + Kimi K2.6 -67.2 +67.0 $0.391 — - + GPT-5.4 nano -65.6 +65.4 $0.002 6 s @@ -1372,7 +1392,7 @@

Cost and latency

Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5’s usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

Simple baselines

-

Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 69.9% on the headline exact-match rate — below the 86% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 25.6 points, and the within-1% companion shows the same separation with near-miss tolerance.

+

Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 69.7% on the headline exact-match rate — below the 85% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 25.5 points, and the within-1% companion shows the same separation with near-miss tolerance.

@@ -1410,16 +1430,16 @@

Simple baselines

0 Always zero -69.9 -69.9 -69.9 +69.7 +69.7 +69.7 1 Median reference by output -63.9 -63.9 -66.8 +63.6 +63.6 +66.5 @@ -1469,51 +1489,51 @@

Sensitivity to benchmark view

0 Exact-match headline -GPT-6 Sol (95.6) -Claude Opus 5.5 (94.4) -GPT-5.6 Sol (93.5) +GPT-6 Sol (95.2) +Claude Opus 5.5 (94.1) +GPT-5.6 Sol (93.1) 1 Within-1% companion -GPT-6 Sol (96.2) -Claude Opus 5.5 (95.8) -GPT-5.6 Sol (94.4) +GPT-6 Sol (95.8) +Claude Opus 5.5 (95.6) +GPT-5.6 Sol (94.0) 2 Bounded score GPT-6 Sol (98.2) Claude Opus 5.5 (98.0) -GPT-6 Luna (97.5) +Muse Wonder (97.6) 3 Equal-output-group score Claude Opus 5.5 (98.3) -GPT-6 Sol (98.3) -GPT-6 Astra (98.1) +GPT-6 Sol (98.2) +GPT-6 Astra (98.0) 4 Amount outputs only -Claude Opus 5.5 (98.0) -Claude Fable 5.1 (97.5) -GPT-6 Astra (97.5) +Claude Opus 5.5 (97.9) +GPT-6 Astra (97.4) +GPT-6.1 Sol (97.4) 5 Positive reference cases only -GPT-6 Astra (95.1) -GPT-6.1 Sol (92.7) -Claude Opus 5.5 (92.3) +GPT-6 Astra (94.9) +GPT-6.1 Sol (92.4) +Claude Opus 5.5 (92.2) 6 Zero reference cases only Claude Opus 5.5 (99.5) GPT-6 Sol (99.5) -Gemini 3.8 Flash (99.2) +Muse Wonder (99.2) @@ -1564,21 +1584,21 @@

Sensitivity to benchmark view

Household-weighted (default) GPT-6 Sol (98.2) Claude Opus 5.5 (98.0) -GPT-6 Luna (97.5) +Muse Wonder (97.6) 1 Equal weights Claude Opus 5.5 (98.4) -GPT-6 Sol (98.3) +GPT-6 Sol (98.1) GPT-6 Astra (97.8) 2 Budget-weighted -GPT-6 Sol (98.0) +GPT-6 Sol (97.9) Claude Opus 5.5 (97.3) -GPT-5.6 Sol (96.9) +Muse Wonder (97.3) @@ -1590,7 +1610,7 @@

Sensitivity to benchmark view

Hardest benchmark targets

-

The lowest-scoring US variables are multi-step tax quantities and sparse positive-dollar benefits. As shown in Table 11, federal income tax before refundable credits, state income tax before refundable credits, state refundable credits, SNAP, and federal refundable credits are the hardest US outputs by bounded score in the frozen manuscript snapshot.

+

The lowest-scoring US variables are multi-step tax quantities and sparse positive-dollar benefits. As shown in Table 11, state income tax before refundable credits, federal income tax before refundable credits, state refundable credits, SNAP, and federal refundable credits are the hardest US outputs by bounded score in the frozen manuscript snapshot.

@@ -1627,44 +1647,44 @@

Hardest benchmark targets -0 -federal_income_tax_before_refundable_credits -82.6 -60.6 -62.9 -71.4 - - 15 state_income_tax_before_refundable_credits -84.4 -64.0 -66.3 -76.5 +82.7 +60.4 +62.7 +73.4 + + +0 +federal_income_tax_before_refundable_credits +82.9 +60.9 +63.4 +71.9 16 state_refundable_credits -86.2 -82.8 -82.8 -83.7 +85.6 +82.1 +82.1 +83.0 13 snap -89.4 -84.2 -85.5 -88.0 +89.5 +84.3 +85.6 +88.1 1 federal_refundable_credits -91.8 -86.9 -88.1 -90.3 +90.8 +85.5 +86.6 +89.0 @@ -1714,50 +1734,50 @@

Zero and positive cases

0 federal_income_tax_before_refundable_credits -72.3 -43.7 -93.4 +72.7 +44.6 +93.6 1 state_income_tax_before_refundable_credits -77.3 -50.3 -92.0 +74.3 +47.3 +91.1 2 state_refundable_credits -84.7 -17.9 +84.0 +17.6 98.6 3 snap 88.4 -35.8 -96.8 +36.1 +96.9 4 federal_refundable_credits -91.6 -57.5 -95.9 +90.3 +51.9 +96.0 5 payroll_tax -91.8 -87.7 +92.0 +88.0 98.5 6 person_medicaid_eligible -92.7 -82.8 +92.8 +83.0 95.6 @@ -1787,12 +1807,12 @@

Zero and positive cases

Open-weight time lag

-

The leaderboard also measures time. Following the lag construction the UK AI Security Institute applied to cyber capabilities (UK AI Security Institute 2026), the open-weight lag is the time between the closed frontier first reaching a score and the leading open-weight model matching it. Two cuts of the frozen roster give the trajectory. Among models released before July 2026, the leading open-weight model is DeepSeek V4 Pro at 79.7% exact, whose weights shipped on April 24, 2026 (DeepSeek 2026); the earliest-released closed model above that score is Gemini 3 Flash Preview, released December 17, 2025 (TechCrunch 2025) — a lag of 128 days, about 4.2 months. On the full roster, Kimi K3 at 91.0% exact trails 7 closed models; the earliest released of them, GPT-5.6 Sol, generally available July 9, 2026 (TechCrunch 2026), puts the lag at 7 days by API availability, or 18 days dated to Kimi K3’s July 27 weight release on Hugging Face (Tech Times 2026).

+

The leaderboard also measures time. Following the lag construction the UK AI Security Institute applied to cyber capabilities (UK AI Security Institute 2026), the open-weight lag is the time between the closed frontier first reaching a score and the leading open-weight model matching it. Two cuts of the frozen roster give the trajectory. Among models released before July 2026, the leading open-weight model is DeepSeek V4 Pro at 79.4% exact, whose weights shipped on April 24, 2026 (DeepSeek 2026); the earliest-released closed model above that score is Gemini 3 Flash Preview, released December 17, 2025 (TechCrunch 2025) — a lag of 128 days, about 4.2 months. On the full roster, Kimi K3 at 90.6% exact trails 7 closed models; the earliest released of them, GPT-5.6 Sol, generally available July 9, 2026 (TechCrunch 2026), puts the lag at 7 days by API availability, or 18 days dated to Kimi K3’s July 27 weight release on Hugging Face (Tech Times 2026).

Both figures are lower bounds in one direction: a closed model outside the evaluated roster that crossed either score earlier would lengthen the corresponding lag. They also carry the uncertainty of any single-benchmark, single-domain estimate — the AISI analysis reports substantially longer lags on cyber tasks — and a lag measured at a moving frontier compresses mechanically when a strong open-weight release lands days after a frontier release.

Failure modes

The benchmark surfaces a few recurring failure patterns.

First, models miss positive tax and benefit quantities more often than zero cases. The lowest-scoring outputs by bounded score (Table 11) require the model to choose the right income concepts, exclusions, program thresholds, and sequencing before applying any final subtraction. Positive benefit cases remain difficult, so the result should not be read as a general claim that benefits are easy.

-

Second, joint accuracy across interacting components can be lower than marginal accuracy on either component. Table 13 shows within-10% accuracy for federal_refundable_credits, state_refundable_credits, and the conjunction of both within the same household. The joint hit rate can be no higher than either marginal and is strictly lower than both for every model except Claude Fable 5.1, GPT-6.1 Sol, Claude Opus 5.5, GPT-6 Astra, GPT-6 Sol, Grok 4.7, and GPT-5.6 Sol. Leaderboard scores that average across outputs therefore overstate how often a model gets a single household’s federal/state credit allocation jointly correct.

+

Second, joint accuracy across interacting components can be lower than marginal accuracy on either component. Table 13 shows within-10% accuracy for federal_refundable_credits, state_refundable_credits, and the conjunction of both within the same household. The joint hit rate can be no higher than either marginal and is strictly lower than both for every model except GPT-6.1 Sol, Claude Fable 5.1, Claude Opus 5.5, GPT-6 Astra, Muse Wonder, and GPT-6 Sol. Leaderboard scores that average across outputs therefore overstate how often a model gets a single household’s federal/state credit allocation jointly correct.

@@ -1828,333 +1848,340 @@

Failure modes

+34 +GPT-6.1 Sol +98.0 +92.9 +92.9 + + 1 Claude Fable 5.1 100.0 -93.8 -93.8 - - -34 -GPT-6.1 Sol -99.0 -93.8 -93.8 +91.9 +91.9 7 Claude Opus 5.5 100.0 -92.8 -92.8 +91.9 +91.9 -32 -GPT-6 Luna +31 +GPT-6 Astra 99.0 -90.7 -89.7 +87.9 +87.9 -31 -GPT-6 Astra -100.0 -88.7 -88.7 +32 +GPT-6 Luna +97.0 +88.9 +87.9 -33 -GPT-6 Sol +44 +Muse Wonder 99.0 -88.7 -88.7 +87.9 +87.9 +33 +GPT-6 Sol +97.0 +86.9 +86.9 + + 38 Grok 4.7 -99.0 -88.7 -88.7 +97.0 +87.9 +86.9 - + 29 GPT-5.6 Sol -99.0 -87.6 -87.6 +97.0 +86.9 +85.9 - + 27 GPT-5.5 -95.9 -88.7 -86.6 +93.9 +87.9 +84.8 - + 14 DeepSeek V4.1 Flash -95.9 -87.6 -85.6 +93.9 +86.9 +83.8 - + 22 Gemini 3.8 Flash -95.9 -87.6 -85.6 +93.9 +86.9 +83.8 - + 37 Grok 4.6 -96.9 -86.6 -85.6 +94.9 +85.9 +83.8 - + 13 DeepSeek V4 Pro 0813 -94.8 -85.6 -83.5 +93.9 +83.8 +81.8 - + 20 Gemini 3.6 Flash -95.9 -84.5 -83.5 +93.9 +82.8 +81.8 - + 21 Gemini 3.7 Flash -91.8 -86.6 -82.5 +89.9 +85.9 +80.8 - + 40 Inkling -94.8 -84.5 -82.5 - - -42 -Kimi K3 -94.8 -83.5 -81.4 +93.9 +83.8 +80.8 0 Claude Fable 5 -92.8 -85.6 -80.4 +91.9 +84.8 +79.8 + + +15 +Gemini 3 Flash Preview +90.9 +84.8 +79.8 + + +42 +Kimi K3 +92.9 +81.8 +79.8 10 Claude Sonnet 5.5 -99.0 -81.4 -80.4 +97.0 +79.8 +78.8 12 DeepSeek V4 Pro -91.8 -84.5 -80.4 +89.9 +82.8 +78.8 -15 -Gemini 3 Flash Preview -91.8 -85.6 -80.4 - - 28 GPT-5.6 Luna -94.8 -82.5 -80.4 +92.9 +81.8 +78.8 - + 30 GPT-5.6 Terra -95.9 -82.5 -80.4 +93.9 +80.8 +78.8 - + 36 Grok 4.5 -93.8 -82.5 -80.4 +92.9 +80.8 +78.8 - + 17 Gemini 3.1 Pro Preview -91.8 -83.5 -79.4 +89.9 +82.8 +77.8 - -44 + +45 GLM-5.3-Flash (preview) -91.8 -84.5 -79.4 +89.9 +83.8 +77.8 - + 18 Gemini 3.5 Flash -91.8 -82.5 -78.4 +89.9 +81.8 +76.8 - + 3 Claude Haiku 5.5 -88.7 -82.5 -77.3 - - -8 -Claude Sonnet 4.6 -93.8 -82.5 -77.3 +86.9 +81.8 +75.8 6 Claude Opus 5 -88.7 -83.5 -76.3 +87.9 +82.8 +75.8 +8 +Claude Sonnet 4.6 +91.9 +81.8 +75.8 + + 11 DeepSeek V4 Flash 0731 -89.7 -83.5 -76.3 +87.9 +81.8 +74.7 - + 16 Gemini 3.1 Flash Lite Preview -86.6 -82.5 -76.3 +84.8 +81.8 +74.7 - + 19 Gemini 3.5 Flash-Lite -87.6 -82.5 -76.3 +85.9 +81.8 +74.7 - + 26 GPT-5.4 nano -87.6 -82.5 -76.3 +85.9 +81.8 +74.7 - + 39 Grok Build 0.1 -90.7 -81.4 -75.3 +89.9 +80.8 +73.7 - + 5 Claude Opus 4.8 -85.6 -82.5 -74.2 +83.8 +81.8 +72.7 - + 35 Grok 4.3 -85.6 -82.5 -74.2 +83.8 +81.8 +72.7 - + 25 GPT-5.4 mini -82.5 -82.5 -73.2 +80.8 +81.8 +71.7 - + 24 GLM-5.3 -87.6 -79.4 -72.2 +85.9 +77.8 +70.7 - + 9 Claude Sonnet 5 -81.4 -81.4 -71.1 +79.8 +80.8 +69.7 - + 43 MiniMax M3 -81.4 -82.5 -71.1 +79.8 +81.8 +69.7 - + 4 Claude Opus 4.7 -86.6 -79.4 -69.1 +84.8 +78.8 +67.7 - + 2 Claude Haiku 4.5 -76.3 -82.5 -68.0 +74.7 +81.8 +66.7 - + 23 GLM-5.2 -81.4 -75.3 -68.0 - - -45 -Qwen3.7-max -79.4 -82.5 -66.0 +79.8 +74.7 +66.7 46 -Qwen3.8-Max -68.0 -82.5 -59.8 +Qwen3.7-max +77.8 +81.8 +64.6 +47 +Qwen3.8-Max +66.7 +81.8 +58.6 + + 41 Kimi K2.6 -46.4 -51.5 -32.0 +46.5 +50.5 +31.3 @@ -2165,19 +2192,19 @@

Failure modes

-

Third, structured-output reliability is part of the benchmark contract. Missing or unparseable numeric values are not dropped. Appendix A documents parser recovery and the supervised runner’s bounded retry rounds. The 659 cells (0.7%) that still returned no parsable value after retries remain in the canonical files and score zero as parse-contract failures. Appendix A gives the per-model breakdown.

+

Third, structured-output reliability is part of the benchmark contract. Missing or unparseable numeric values are not dropped. Appendix A documents parser recovery and the supervised runner’s bounded retry rounds. The 669 cells (0.7%) that still returned no parsable value after retries remain in the canonical files and score zero as parse-contract failures. Appendix A gives the per-model breakdown.

Limitations

PolicyBench is not a substitute for a production tax-and-benefit calculator. Several caveats matter:

  • output-contract reliability required a repair workflow; raw provider responses are retained within the canonical prediction files wherever the transport exposed them (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)), and cells that never parsed score zero
    • -
  • request shape is not identical across models: 10 of 47 models answered the prompt template over subsets of the requested outputs rather than the whole scenario (Table 2), an accommodation that, if it moves scores at all, should favor those models; later additions must answer the whole-scenario request or are listed as not scorable
    • +
  • request shape is not identical across models: 10 of 48 models answered the prompt template over subsets of the requested outputs rather than the whole scenario (Table 2), an accommodation that, if it moves scores at all, should favor those models; later additions must answer the whole-scenario request or are listed as not scorable
  • zero-heavy outputs require separate positive-case interpretation
  • the choice of headline metric (exact match), near-miss companion (within-1%), and secondary tracking metric (bounded score) is one benchmark view; all three are reported, but downstream evaluations should also consider error-magnitude metrics (mean absolute error, mean absolute percentage error) where appropriate
  • the public scenario explorer exposes the current test set and reference outputs, so open-set leakage is a prominent limitation rather than a minor implementation detail
  • the 100-household manuscript snapshot should be treated as a preview until larger frozen runs are published
    • -
  • the annotations cover the 8,001-row legacy-threshold audit universe, including 7,997 exact-match misses and 4 exact hits, but not 2,138 additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
    • +
  • the annotations cover the 8,503-row legacy-threshold audit universe, including 8,499 exact-match misses and 4 exact hits, but not 2,218 additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
  • one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (disability, which the prompt states as one general fact where each program applies its own determination, and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address (Section 4.2); outputs whose reference moves under the other reading of either convention are excluded, while the mis-encoded scenario’s cells score models against its encoded inputs
    • -
  • the reference engine carries known defects: 10 outputs, recorded as engine-defect exclusions, rest on 6 root causes that the audits found in policyengine-us 1.755.4 or later releases and that 2.38.6 does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, each engine-upgrade review covered only the outputs its upgrade moved, and on 2.15.17 PolicyBench re-ran four of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
    • +
  • the reference engine can carry defects the audits have not found: policyengine-us 2.41.0 no longer has any defect the audits recorded behind an excluded output; the judge that finds defects reviews only the cases a new model joins, each engine-upgrade review covered only the outputs its upgrade moved, and on 2.15.17 PolicyBench re-ran four of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
  • the 2026-10-05 review of release dashboard-data-20260930 found eight scored references that rested on inputs the prompt never states and that no judge had flagged, so other such references can remain scored: the exclusions cover the inputs the audits found
  • the references were frozen on 2026-07-03 for tax year 2026, so amounts published later reach the benchmark only through a stated convention: where policyengine-us projected a 2026 amount, the reference takes the amount published before the freeze or, failing that, the last one published (SNAP’s FY2026 figures rather than USDA’s FY2027 figures of 2026-08-21; California’s 2025 indexed amounts; the IRS’s 2025 sales tax tables), so a model that knew later law is scored against the earlier law, and the projection audit sourced only the amounts whose projection moved an output; law published before the freeze reaches a reference only once policyengine-us encodes it, and policyengine-us encoded New Jersey’s 2026 child tax credit schedule and Arizona’s SNAP categorical eligibility limit only after 1.755.4
  • benchmark success should not be interpreted as policy-advice readiness
    • @@ -2223,7 +2250,7 @@

    Appendix Final parse coverage -659 of the 90,522 canonical model-output rows (0.7%) have no parsable numeric value after retries. The failures comprise Kimi K2.6 (390), GLM-5.2 (136), GLM-5.3 (73), and Kimi K3 (60). They score zero as parse-contract failures. A further 59 rows (Grok 4.3 (55), Kimi K2.6 (3), and Claude Haiku 4.5 (1)) carry a parsed numeric value but no explanation; they are scored on the numeric value and counted here as explanation-contract violations, so 718 rows in all fall short of the numeric-plus-explanation contract. Every other row has both. +669 of the 92,880 canonical model-output rows (0.7%) have no parsable numeric value after retries. The failures comprise Kimi K2.6 (397), GLM-5.2 (136), GLM-5.3 (75), and Kimi K3 (61). They score zero as parse-contract failures. A further 59 rows (Grok 4.3 (55), Kimi K2.6 (3), and Claude Haiku 4.5 (1)) carry a parsed numeric value but no explanation; they are scored on the numeric value and counted here as explanation-contract violations, so 728 rows in all fall short of the numeric-plus-explanation contract. Every other row has both. Preservation rule @@ -2236,7 +2263,7 @@

    Appendix

-

Model waves after June ran under a resumable supervised runner that retries failed or timed-out scenarios in bounded rounds. No separate retry artifacts exist; repairs are folded into each model’s canonical prediction file. After retries, 659 cells (0.7% of canonical rows) still returned no parsable numeric value. They comprise Kimi K2.6 (390), GLM-5.2 (136), GLM-5.3 (73), and Kimi K3 (60). These cells remain in the canonical files, with their raw responses preserved wherever the transport exposed one (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)). They score zero as parse-contract failures, and the denominator is never reduced.

+

Model waves after June ran under a resumable supervised runner that retries failed or timed-out scenarios in bounded rounds. No separate retry artifacts exist; repairs are folded into each model’s canonical prediction file. After retries, 669 cells (0.7% of canonical rows) still returned no parsable numeric value. They comprise Kimi K2.6 (397), GLM-5.2 (136), GLM-5.3 (75), and Kimi K3 (61). These cells remain in the canonical files, with their raw responses preserved wherever the transport exposed one (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)). They score zero as parse-contract failures, and the denominator is never reduced.

Competing interests

The author leads PolicyEngine, which develops the microsimulation software used to produce the benchmark reference outputs, and is a founder of the Axiom Foundation, a nonprofit developing deterministic rules-as-code infrastructure. Both organizations’ missions are advanced by the finding that unaided language models do not reliably reproduce rule calculations.

diff --git a/app/src/components/ProviderMark.tsx b/app/src/components/ProviderMark.tsx index 1bf65ea8..d5678593 100644 --- a/app/src/components/ProviderMark.tsx +++ b/app/src/components/ProviderMark.tsx @@ -3,6 +3,7 @@ import { Anthropic, DeepSeek, Google, + Meta, Minimax, Moonshot, OpenAI, @@ -78,6 +79,7 @@ export default function ProviderMark({ {provider === "anthropic" &&