diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7add0f6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +* +!pyproject.toml +!src/ +src/* +!src/marginal/ +!src/marginal/** +!benchmark/ +benchmark/* +!benchmark/__init__.py +!benchmark/codex_adapter/ +!benchmark/codex_adapter/** +!benchmark/container/ +!benchmark/container/** diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 99f457a..618e296 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -44,6 +44,7 @@ jobs: test -f _site/index.html test -f _site/styles.css test -f _site/app.js + test -f _site/benchmark-data.json test -f _site/robots.txt test -f _site/sitemap.xml test -f _site/assets/marginal-readme-hero.png diff --git a/.github/workflows/swebench-lite-canary.yml b/.github/workflows/swebench-lite-canary.yml index 5916739..be21dd1 100644 --- a/.github/workflows/swebench-lite-canary.yml +++ b/.github/workflows/swebench-lite-canary.yml @@ -58,7 +58,12 @@ jobs: set -euo pipefail python -m pip install --upgrade pip python -m pip install -e ".[dev]" - python -m pip install modal "swebench[modal]" + python -m pip install "modal==1.5.3" "swebench==4.1.0" + python -m modal token set \ + --token-id "${MODAL_TOKEN_ID}" \ + --token-secret "${MODAL_TOKEN_SECRET}" \ + --verify + test -s ~/.modal.toml - name: Validate evidence before Modal spend shell: bash @@ -78,14 +83,15 @@ jobs: set -euo pipefail IDS="$(python benchmarks/swebench_lite/protocol.py ids --task-set "${TASK_SET}")" rm -rf evaluation_results + cp "${RUN_DIR}/baseline_predictions.ndjson" /tmp/baseline_predictions.jsonl RUN_ID="marginal-${TASK_SET}-baseline-${GITHUB_RUN_ID}" python -m swebench.harness.run_evaluation \ --dataset_name princeton-nlp/SWE-bench_Lite \ --split dev \ - --predictions_path "${RUN_DIR}/baseline_predictions.ndjson" \ + --predictions_path /tmp/baseline_predictions.jsonl \ --instance_ids ${IDS} \ --run_id "${RUN_ID}" \ - --parallelism 10 \ + --max_workers 10 \ --modal true RESULT="$(find evaluation_results -type f -name instance_results.jsonl -print -quit 2>/dev/null || true)" if [ -n "${RESULT}" ]; then @@ -102,14 +108,15 @@ jobs: set -euo pipefail IDS="$(python benchmarks/swebench_lite/protocol.py ids --task-set "${TASK_SET}")" rm -rf evaluation_results + cp "${RUN_DIR}/marginal_predictions.ndjson" /tmp/marginal_predictions.jsonl RUN_ID="marginal-${TASK_SET}-on-${GITHUB_RUN_ID}" python -m swebench.harness.run_evaluation \ --dataset_name princeton-nlp/SWE-bench_Lite \ --split dev \ - --predictions_path "${RUN_DIR}/marginal_predictions.ndjson" \ + --predictions_path /tmp/marginal_predictions.jsonl \ --instance_ids ${IDS} \ --run_id "${RUN_ID}" \ - --parallelism 10 \ + --max_workers 10 \ --modal true RESULT="$(find evaluation_results -type f -name instance_results.jsonl -print -quit 2>/dev/null || true)" if [ -n "${RESULT}" ]; then @@ -162,7 +169,7 @@ jobs: - name: Upload benchmark evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: swebench-lite-${{ inputs.task_set }}-${{ github.run_id }} if-no-files-found: warn diff --git a/.gitignore b/.gitignore index ad04826..388b40c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ build/ # MARGINAL local privacy keys *.privacy.key .marginal/ + +# Local Codex benchmark trajectories, task checkouts, and copied runtime state +/benchmark/runs/ +/benchmark/.venv/ + +# Aggregated benchmark records are intentionally reviewable artifacts. +!/benchmark/results/*.jsonl diff --git a/README.md b/README.md index 51116bd..e7da414 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,31 @@ Open source · Local first · Provider neutral · Zero mandatory runtime depende --- +## First measured Codex integration — correctness first + +> **Exploratory 3-task smoke, one paired run per task.** This validates the integration; it is not a general performance claim. + +| Metric | Codex OFF | Codex + MARGINAL | Observed change | +|---|---:|---:|---:| +| SWE-bench resolved | 0/3 | 0/3 | **0/3 → 0/3** | +| Effective tokens | 1,098,747 | 824,839 | **24.93% fewer** | +| Effective latency | 593.11 s | 565.77 s | **4.61% lower** | +| Tool calls | 33 | 32 | **3.03% fewer** | +| Governance overhead | — | 0 tokens · $0 · **7.06 s** | measured separately | +| Evaluator decision | — | **pass_through** | not eligible for a support claim | + +The matched trajectories used Codex CLI 0.147.0, GPT-5.6-sol, identical prompts and limits, and pinned SWE-bench Lite task images. The official verifier completed all six evaluations without infrastructure errors in the authoritative Docker run. Neither lane resolved a task, so tokens per resolved task is undefined: the observed 24.93% token difference is useful trajectory telemetry, not proof of useful token saving. **No deny was applied in these three agent trajectories**, so the difference cannot be attributed to a MARGINAL stop decision. + +The Modal audit completed two tasks per lane but hit the same image-build error for `pylint-dev__astroid-1978` in both lanes. Those cloud errors are preserved and excluded from scoring; the public evaluator now fails closed on `error_ids` and `incomplete_ids` instead of silently treating them as unresolved. + +[Full report](benchmarks/swebench_lite/PUBLIC_BENCHMARK.md) · +[Raw JSON](benchmarks/swebench_lite/public-benchmark.json) · +[Evidence bundle](benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/) · +[Frozen protocol](benchmarks/swebench_lite/README.md) · +[Modal workflow run](https://github.com/SignalLayerLabs/Marginal/actions/runs/31474500980) + +--- + ## The problem in one trace Coding agents can spend compute on actions whose incremental value is unclear or diminishing. The useful failure mode is not "GPT-5.6 is wasteful"; it is **repeated work against unchanged state that produces no new evidence**. @@ -174,7 +199,7 @@ cd Marginal python -m pip install -e ".[dev]" ``` -The upcoming Codex reference integration remains a roadmap milestone. It is not presented here as already available. +The auditable Codex reference adapter and its first matched smoke are now available in `benchmark/codex_adapter/`. Start from the frozen protocol and treat the current n=3 result as integration evidence, not a performance claim. ## Quickstart diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..0f47a47 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,77 @@ +# Codex baseline versus Codex + MARGINAL + +This directory contains the preregistration, runner, raw evidence, and analysis for a +correctness-first paired evaluation of Codex with MARGINAL OFF and ON. + +No synthetic MARGINAL demo result is accepted as evidence for this comparison. Task +correctness comes only from the benchmark's independent verifier. Failed, timed-out, +empty-patch, and integration-failed runs remain in the dataset. + +## Frozen smoke set + +- Dataset: `princeton-nlp/SWE-bench_Lite` +- Split: `dev` +- Selection rule: the three lexicographically lowest SHA-256 digests of `instance_id` +- Task-set digest: `1c8c06935484fd57f4c27c363418884542bcfb06c69d02f480b61fe7687b58eb` +- Repetitions: one engineering run per condition +- Conditions: `baseline` and `marginal` + +The smoke is an integration check, not publication-grade performance evidence. The +20-task canary and larger repeated experiment remain gated on complete smoke telemetry. + +## Commands + +Run the non-verifier integration gate (this does not launch Codex inference): + +```bash +.venv/bin/python -m benchmark.scripts.preflight --skip-verifier +``` + +Run the mandatory full gate before any benchmark trajectory: + +```bash +.venv/bin/python -m benchmark.scripts.preflight +``` + +The full gate currently stops before inference for two scientific blockers: + +- Codex 0.147 does not expose shell exit status in `PostToolUse`, so the preregistered + successful shell-repeat mechanism cannot distinguish passing from failing commands; and +- Codex is not yet running inside a pinned official per-instance SWE-bench environment. + +An official verifier backend is also required after those blockers are resolved. The task +materializer and single-run adapter remain available for test-only integration work, but +must not be used for benchmark inference on a bare host checkout. + +The future matched layout uses distinct worktrees for each lane: + +```bash +.venv/bin/python -m benchmark.scripts.prepare_swebench_tasks \ + --destination benchmark/runs/worktrees/baseline +.venv/bin/python -m benchmark.scripts.prepare_swebench_tasks \ + --destination benchmark/runs/worktrees/marginal +.venv/bin/python -m benchmark.scripts.run_codex_task \ + --instance-id pvlib__pvlib-python-1072 \ + --condition baseline \ + --worktree benchmark/runs/worktrees/baseline/pvlib__pvlib-python-1072 \ + --run-dir benchmark/runs/pvlib__pvlib-python-1072/baseline +``` + +Run the MARGINAL condition against its separate checkout under +`benchmark/runs/worktrees/marginal/`, then repeat for the other frozen instance IDs. The +runner rejects a dirty tree, an attached branch, or a commit other than the frozen base. +Do not begin the 20-task canary until all six smoke trajectories have complete raw +telemetry and independent verifier results. + +The frozen design and implementation plan are under `docs/superpowers/`. The full +preflight remains the authority: do not launch a trajectory unless it passes. + +## Safety + +- Do not place API keys, Codex authentication files, prompts containing secrets, source + content, or raw tool output in tracked results. +- The runner isolates `HOME`, gives tool subprocesses an allowlisted environment, keeps + `CODEX_HOME` out of that environment, and fails closed if exact authentication material + appears in any changed file or extracted patch. +- Local runtime data belongs under `benchmark/runs/` and is ignored. +- Do not commit or push benchmark work unless explicitly authorized. diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100644 index 0000000..4395dae --- /dev/null +++ b/benchmark/__init__.py @@ -0,0 +1 @@ +"""Reproducible Codex benchmark integration for MARGINAL.""" diff --git a/benchmark/analysis/failure_analysis.md b/benchmark/analysis/failure_analysis.md new file mode 100644 index 0000000..90e8261 --- /dev/null +++ b/benchmark/analysis/failure_analysis.md @@ -0,0 +1,17 @@ +# Smoke Failure Analysis + +No trajectories are available yet. Observations and interpretations will be kept separate. + +## Preflight blocker + +- Observation: Codex 0.147 `PostToolUse` exposes shell output but not exit status. Treating + every completed shell handler as successful would corrupt MARGINAL's successful-repeat + history; treating every shell as failed makes the preregistered primary intervention + effectively pass-through for searches and tests. +- Observation: task checkouts are materialized at the correct commits, but Codex is not + yet executed inside the official pinned per-instance SWE-bench runtime. +- Observation: no official verifier backend is available on this host. +- Interpretation: executing trajectories now would measure an unfaithful mechanism in an + uncontrolled task environment and would spend tokens without admissible correctness. +- Resolution: introduce a tested success-observable interception boundary, run Codex in + the official task environment, configure the official verifier, and rerun full preflight. diff --git a/benchmark/analysis/summary.md b/benchmark/analysis/summary.md new file mode 100644 index 0000000..2104f5a --- /dev/null +++ b/benchmark/analysis/summary.md @@ -0,0 +1,81 @@ +# Codex OFF versus Codex + MARGINAL: Smoke Report + +## Executive Summary + +No Codex OFF/ON trajectory was executed. The integration-only preflight passes for the +frozen three-task set, Codex 0.147.0, `gpt-5.6-sol`, and `high` reasoning effort. The +mandatory full preflight stops before inference because successful shell executions are +not observable through the pinned hook contract and no official per-instance SWE-bench +execution backend is implemented. An official verifier is also unavailable on this host. + +Running anyway would spend tokens on an experiment that cannot faithfully test the frozen +mechanism or produce admissible correctness. There are therefore no performance claims. + +## What MARGINAL Actually Does + +MARGINAL is a provider-neutral action-governance library built around a `Treasury`, policy, +budgets, decision ledger, and optional state-aware diminishing-return detector. It does not +natively modify Codex prompts or model calls. The benchmark adapter maps official Codex +tool hooks to MARGINAL actions and can deny a proposed action in enforce mode. The baseline +has no MARGINAL process, hook, state, or context. + +## Methodology + +The preregistered smoke uses three deterministically selected SWE-bench Lite `dev` tasks, +one run per condition, separate exact-base-commit checkouts, identical frozen prompt/model/ +reasoning/sandbox/timeout settings, raw JSONL telemetry, and the official SWE-bench +resolution criterion. Correctness is primary. The smoke is an integration check and does +not support population-level performance inference. + +## Correctness Results + +| Metric | Codex baseline | Codex + MARGINAL | Delta | +|---|---:|---:|---:| +| Executed tasks | 0 | 0 | 0 | +| Resolved tasks | unavailable | unavailable | unavailable | +| Resolution rate | unavailable | unavailable | unavailable | + +## Paired Results + +- `BOTH_SOLVE`: unavailable +- `BASELINE_ONLY`: unavailable +- `MARGINAL_ONLY`: unavailable +- `NEITHER`: unavailable + +## Efficiency + +Tokens, time, actions, tool calls, repeated behavior, files modified, and diff size are all +unavailable because no comparative trajectory was launched. + +## Intervention Analysis + +No benchmark intervention occurred. Beneficial, neutral, harmful, indeterminate, and +false-positive rates are unavailable. + +## Regressions + +No regression was observed because no paired outcome exists. This is not evidence of no +regressions. + +## Failure Analysis + +The blocking observations and their interpretation are recorded in `failure_analysis.md`. +They are infrastructure and measurement failures, not model/task outcomes. + +## Limitations + +- Codex 0.147 unified-shell `PostToolUse` omits exit status, preventing defensible + successful-repeat history for the main search/test/debugging surface. +- Codex is not provisioned inside an official pinned per-instance SWE-bench environment. +- No official verifier backend is available locally. +- The frozen set contains only three tasks and one repetition even after feasibility is + restored; any future smoke result will remain exploratory. +- Agent and environment nondeterminism require larger repeated paired runs for credible + generalization. + +## Conclusion + +The evidence is insufficient to answer whether MARGINAL improves task completion, reduces +unproductive behavior or cost, intervenes incorrectly, helps or hurts particular task +classes, or generalizes beyond stress tests. The current evidence is not strong enough to +justify enabling MARGINAL with Codex on the basis of this benchmark. diff --git a/benchmark/codex_adapter/__init__.py b/benchmark/codex_adapter/__init__.py new file mode 100644 index 0000000..73204ba --- /dev/null +++ b/benchmark/codex_adapter/__init__.py @@ -0,0 +1 @@ +"""Codex hook adapter components used by the benchmark harness.""" diff --git a/benchmark/codex_adapter/codex_events.py b/benchmark/codex_adapter/codex_events.py new file mode 100644 index 0000000..4451bef --- /dev/null +++ b/benchmark/codex_adapter/codex_events.py @@ -0,0 +1,191 @@ +"""Strict accounting over the stable ``codex exec --json`` event stream.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_TEST_COMMAND = re.compile( + r"(?:^|[;&|]\s*)(?:\S*/)?(?:pytest|py\.test|tox|nox|go\s+test|cargo\s+test|" + r"npm\s+(?:run\s+)?test|pnpm\s+(?:run\s+)?test|yarn\s+(?:run\s+)?test|" + r"python(?:\d+(?:\.\d+)?)?\s+-m\s+(?:pytest|unittest))\b", + re.IGNORECASE, +) +_SEARCH_COMMAND = re.compile( + r"(?:^|[;&|]\s*)(?:\S*/)?(?:rg|grep|find|fd)(?:\s|$)|" + r"(?:^|[;&|]\s*)git\s+grep(?:\s|$)", + re.IGNORECASE, +) + + +class EventParseError(ValueError): + """Raised when raw Codex telemetry cannot support auditable metrics.""" + + +@dataclass(frozen=True, slots=True) +class CodexMetrics: + thread_id: str | None + completed: bool + tokens: dict[str, int] | None + tool_calls: int + shell_commands: int + file_operations: int + searches: int + test_executions: int + repeated_calls: int + errors: tuple[str, ...] + + +def _canonical(value: object) -> str: + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError) as exc: + raise EventParseError("event contains non-canonical JSON") from exc + + +def _usage(event: dict[str, Any]) -> dict[str, int]: + usage = event.get("usage") + if not isinstance(usage, dict): + raise EventParseError("turn.completed is missing usage") + fields = { + "input": "input_tokens", + "cached_input": "cached_input_tokens", + "output": "output_tokens", + "reasoning": "reasoning_output_tokens", + } + result: dict[str, int] = {} + for output_name, input_name in fields.items(): + value = usage.get(input_name) + if isinstance(value, bool) or not isinstance(value, int): + raise EventParseError(f"{input_name} must be an integer") + if value < 0: + raise EventParseError("token usage must be non-negative") + result[output_name] = value + if result["cached_input"] > result["input"]: + raise EventParseError("cached input must be a subset of input tokens") + if result["reasoning"] > result["output"]: + raise EventParseError("reasoning output must be a subset of output tokens") + # Codex reports cached input as a subset of input and reasoning as a subset of output. + result["total"] = result["input"] + result["output"] + return result + + +def _identity(item: dict[str, Any]) -> str | None: + item_type = item.get("type") + if item_type == "command_execution": + command = item.get("command") + return f"command:{command}" if isinstance(command, str) else None + if item_type == "file_change": + return f"file_change:{_canonical(item.get('changes'))}" + if item_type == "mcp_tool_call": + return "mcp:" + _canonical( + { + "server": item.get("server"), + "tool": item.get("tool"), + "arguments": item.get("arguments"), + } + ) + if item_type == "web_search": + return f"web_search:{item.get('query')}" + if item_type == "collab_tool_call": + return "collab:" + _canonical({"tool": item.get("tool"), "prompt": item.get("prompt")}) + return None + + +def parse_codex_jsonl(path: str | Path) -> CodexMetrics: + """Parse one complete run without guessing missing telemetry.""" + + event_path = Path(path) + thread_id: str | None = None + turn_started = False + completed = False + tokens: dict[str, int] | None = None + tool_calls = 0 + shell_commands = 0 + file_operations = 0 + searches = 0 + test_executions = 0 + repeated_calls = 0 + identities: set[str] = set() + errors: list[str] = [] + + with event_path.open(encoding="utf-8") as stream: + for line_number, raw in enumerate(stream, start=1): + if not raw.strip(): + continue + try: + event = json.loads(raw) + except json.JSONDecodeError as exc: + raise EventParseError(f"invalid JSON on line {line_number}: {exc.msg}") from exc + if not isinstance(event, dict): + raise EventParseError(f"event on line {line_number} must be an object") + event_type = event.get("type") + if event_type == "thread.started": + value = event.get("thread_id") + if isinstance(value, str) and value: + thread_id = value + elif event_type == "turn.started": + turn_started = True + elif event_type == "turn.completed": + if tokens is not None: + raise EventParseError("multiple turn.completed events are unsupported") + tokens = _usage(event) + completed = True + elif event_type == "turn.failed": + error = event.get("error") + message = error.get("message") if isinstance(error, dict) else None + errors.append(str(message or "turn failed")) + elif event_type == "error": + errors.append(str(event.get("message") or "unknown stream error")) + elif event_type != "item.completed": + continue + + if event_type != "item.completed": + continue + item = event.get("item") + if not isinstance(item, dict): + raise EventParseError(f"item.completed on line {line_number} has no item object") + identity = _identity(item) + if identity is None: + continue + tool_calls += 1 + if identity in identities: + repeated_calls += 1 + identities.add(identity) + + item_type = item.get("type") + if item_type == "command_execution": + shell_commands += 1 + command = item.get("command") + if isinstance(command, str): + searches += int(bool(_SEARCH_COMMAND.search(command))) + test_executions += int(bool(_TEST_COMMAND.search(command))) + elif item_type == "file_change": + file_operations += 1 + elif item_type == "web_search": + searches += 1 + + if completed and (thread_id is None or not turn_started): + raise EventParseError("completed turn has an invalid thread/turn lifecycle") + + return CodexMetrics( + thread_id=thread_id, + completed=completed, + tokens=tokens, + tool_calls=tool_calls, + shell_commands=shell_commands, + file_operations=file_operations, + searches=searches, + test_executions=test_executions, + repeated_calls=repeated_calls, + errors=tuple(errors), + ) diff --git a/benchmark/codex_adapter/container_runner.py b/benchmark/codex_adapter/container_runner.py new file mode 100644 index 0000000..72dcd8a --- /dev/null +++ b/benchmark/codex_adapter/container_runner.py @@ -0,0 +1,373 @@ +"""Run one Codex lane in a pinned SWE-bench container and attest its evidence.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +import time +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .codex_events import EventParseError, parse_codex_jsonl +from .container_runtime import ContainerRunConfig, build_container_command + + +class ContainerTaskError(RuntimeError): + """Raised when a run cannot satisfy the frozen integrity contract.""" + + +@dataclass(frozen=True, slots=True) +class ContainerTaskConfig: + instance_id: str + condition: str + repetition: int + expected_base_commit: str + task_image: str + overlay_image: str + run_dir: Path + source_root: Path + auth_source: Path + prompt: str + source_commit: str + model: str = "gpt-5.6-sol" + reasoning_effort: str = "high" + codex_version: str = "0.147.0" + timeout_seconds: int = 1800 + + def __post_init__(self) -> None: + if self.repetition < 1: + raise ValueError("repetition must be positive") + if not self.prompt.strip(): + raise ValueError("prompt must not be empty") + if re.fullmatch(r"[0-9a-f]{40}", self.source_commit) is None: + raise ValueError("source_commit must be a lowercase 40-character SHA") + + +Executor = Callable[..., subprocess.CompletedProcess[Any]] + + +def _stdout_text(completed: subprocess.CompletedProcess[Any]) -> str: + output = completed.stdout + if isinstance(output, bytes): + return output.decode("utf-8", errors="replace") + return str(output or "") + + +def _inspect_runtime(config: ContainerTaskConfig, executor: Executor) -> None: + try: + info = executor( + ["docker", "info", "--format", "{{.Architecture}}"], + check=False, + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ContainerTaskError(f"container engine inspection failed: {exc}") from exc + if info.returncode != 0 or _stdout_text(info).strip() != "x86_64": + raise ContainerTaskError("container engine must expose an x86_64 Docker runtime") + + inspected: dict[str, dict[str, Any]] = {} + for role, image in (("task", config.task_image), ("overlay", config.overlay_image)): + completed = executor( + ["docker", "image", "inspect", image], + check=False, + capture_output=True, + text=True, + timeout=60, + ) + try: + values = json.loads(_stdout_text(completed)) + except json.JSONDecodeError as exc: + raise ContainerTaskError(f"{role} image inspection returned invalid JSON") from exc + if completed.returncode != 0 or not isinstance(values, list) or len(values) != 1: + raise ContainerTaskError(f"{role} image is unavailable or ambiguous") + value = values[0] + if not isinstance(value, dict) or value.get("Architecture") != "amd64": + raise ContainerTaskError(f"{role} image architecture is not amd64") + inspected[role] = value + + task_digests = inspected["task"].get("RepoDigests") + if not isinstance(task_digests, list) or config.task_image not in task_digests: + raise ContainerTaskError("task image digest provenance mismatch") + overlay = inspected["overlay"] + overlay_identity = overlay.get("Id") + overlay_digests = overlay.get("RepoDigests") + if config.overlay_image != overlay_identity and ( + not isinstance(overlay_digests, list) or config.overlay_image not in overlay_digests + ): + raise ContainerTaskError("overlay image digest provenance mismatch") + config_value = overlay.get("Config") + labels = config_value.get("Labels") if isinstance(config_value, dict) else None + if not isinstance(labels, dict): + raise ContainerTaskError("overlay image has no provenance labels") + if labels.get("org.marginal.codex.version") != config.codex_version: + raise ContainerTaskError("overlay Codex version provenance mismatch") + if labels.get("org.marginal.source.commit") != config.source_commit: + raise ContainerTaskError("overlay source commit provenance mismatch") + if labels.get("org.marginal.task.image") != config.task_image: + raise ContainerTaskError("overlay task image provenance mismatch") + + +def _json_object(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContainerTaskError(f"invalid or missing runtime artifact: {path.name}") from exc + if not isinstance(value, dict): + raise ContainerTaskError(f"runtime artifact must be an object: {path.name}") + return value + + +def _secret_markers(path: Path) -> tuple[bytes, ...]: + raw = path.read_bytes() + markers: set[bytes] = {raw.strip()} if len(raw.strip()) >= 16 else set() + try: + decoded = json.loads(raw) + except json.JSONDecodeError: + decoded = None + + def collect(value: object) -> None: + if isinstance(value, str): + encoded = value.encode("utf-8") + if len(encoded) >= 16: + markers.add(encoded) + elif isinstance(value, dict): + for child in value.values(): + collect(child) + elif isinstance(value, list): + for child in value: + collect(child) + + collect(decoded) + return tuple(markers) + + +def _assert_no_auth_material(run_dir: Path, auth_source: Path) -> None: + markers = _secret_markers(auth_source) + for artifact in run_dir.iterdir(): + if not artifact.is_file(): + continue + try: + content = artifact.read_bytes() + except OSError: + continue + if any(marker in content for marker in markers): + raise ContainerTaskError( + f"authentication material detected in runtime artifact: {artifact.name}" + ) + + +def _patch_stats(path: Path) -> tuple[int, int | None]: + files: set[str] = set() + diff_lines = 0 + binary = False + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as exc: + raise ContainerTaskError("invalid or missing runtime artifact: model.numstat") from exc + for line in lines: + fields = line.split("\t", maxsplit=2) + if len(fields) != 3: + raise ContainerTaskError("malformed model.numstat") + added, deleted, filename = fields + files.add(filename) + if added == "-" or deleted == "-": + binary = True + else: + try: + diff_lines += int(added) + int(deleted) + except ValueError as exc: + raise ContainerTaskError("malformed model.numstat") from exc + return len(files), None if binary else diff_lines + + +def _configuration_hash(config: ContainerTaskConfig) -> str: + payload = { + "schema_version": 1, + "instance_id": config.instance_id, + "condition": config.condition, + "repetition": config.repetition, + "base_commit": config.expected_base_commit, + "source_commit": config.source_commit, + "task_image": config.task_image, + "overlay_image": config.overlay_image, + "model": config.model, + "reasoning_effort": config.reasoning_effort, + "codex_version": config.codex_version, + "timeout_seconds": config.timeout_seconds, + "prompt_sha256": hashlib.sha256(config.prompt.encode("utf-8")).hexdigest(), + "sandbox": "workspace-write", + "tool_network_access": False, + "marginal": "absent" if config.condition == "baseline" else "enforce", + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _attest_marginal(run_dir: Path, tool_calls: int) -> tuple[dict[str, int], dict[str, Any]]: + if (run_dir / "hook-failures.log").exists(): + raise ContainerTaskError("hook failure artifact exists") + summary = _json_object(run_dir / "daemon-summary.json") + counts = { + key: int(summary.get(key, -1)) + for key in ("approved", "committed", "denied", "aborted", "failed_settled", "pending") + } + governance = summary.get("governance") + interventions = summary.get("interventions") + if not isinstance(governance, dict) or not isinstance(interventions, dict): + raise ContainerTaskError("daemon summary is missing accounting objects") + decisions = int(governance.get("decisions", -1)) + coverage_ok = ( + counts["pending"] == 0 + and counts["aborted"] == 0 + and counts["failed_settled"] == 0 + and counts["approved"] == counts["committed"] + and counts["committed"] == tool_calls + and decisions == counts["approved"] + counts["denied"] + ) + if not coverage_ok: + raise ContainerTaskError("MARGINAL hook coverage is incomplete or inconsistent") + expected_denies = int(interventions.get("applied_denies", -1)) + if expected_denies != counts["denied"]: + raise ContainerTaskError("MARGINAL denial accounting is inconsistent") + return counts, {"governance": governance, "interventions": interventions} + + +def run_container_task( + config: ContainerTaskConfig, + *, + executor: Executor = subprocess.run, +) -> dict[str, Any]: + """Execute one disposable lane, fail closed, and write a schema-v1 record.""" + + _inspect_runtime(config, executor) + run_dir = config.run_dir.resolve() + if run_dir.exists(): + raise ContainerTaskError(f"run directory already exists: {run_dir}") + run_dir.mkdir(parents=True) + prompt_path = run_dir / "prompt.txt" + prompt_path.write_text(config.prompt, encoding="utf-8") + runtime = ContainerRunConfig( + instance_id=config.instance_id, + condition=config.condition, + expected_base_commit=config.expected_base_commit, + task_image=config.task_image, + overlay_image=config.overlay_image, + container_name=( + f"marginal-{config.condition}-{config.instance_id.replace('__', '-')}-" + f"{config.repetition}" + ).lower(), + run_dir=run_dir, + source_root=config.source_root, + auth_source=config.auth_source, + prompt_file=prompt_path, + model=config.model, + reasoning_effort=config.reasoning_effort, + timeout_seconds=config.timeout_seconds, + ) + started = time.perf_counter_ns() + try: + completed = executor( + build_container_command(runtime), + check=False, + timeout=config.timeout_seconds + 60, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ContainerTaskError(f"container execution failed: {exc}") from exc + wall_time_ms = max(0, round((time.perf_counter_ns() - started) / 1_000_000)) + + status = _json_object(run_dir / "container-status.json") + if status != { + "codex_exit_code": completed.returncode, + "condition": config.condition, + "instance_id": config.instance_id, + }: + raise ContainerTaskError("container exit status provenance mismatch") + if completed.returncode != 0: + raise ContainerTaskError(f"Codex container exited with {completed.returncode}") + try: + metrics = parse_codex_jsonl(run_dir / "codex-events.jsonl") + except (OSError, EventParseError) as exc: + raise ContainerTaskError(f"Codex event stream is invalid: {exc}") from exc + if not metrics.completed or metrics.tokens is None or metrics.errors: + raise ContainerTaskError("Codex event lifecycle or token telemetry is incomplete") + + if config.condition == "marginal": + _, marginal = _attest_marginal(run_dir, metrics.tool_calls) + raw_governance = marginal["governance"] + raw_interventions = marginal["interventions"] + else: + forbidden = ( + "daemon-summary.json", + "treasury-events.jsonl", + "hook-failures.log", + "daemon.stdout.log", + "daemon.stderr.log", + ) + if any((run_dir / name).exists() for name in forbidden): + raise ContainerTaskError("baseline contains a MARGINAL runtime artifact") + raw_governance = {} + raw_interventions = {} + + _assert_no_auth_material(run_dir, config.auth_source) + patch = (run_dir / "model.patch").read_bytes() + files_modified, diff_lines = _patch_stats(run_dir / "model.numstat") + interventions = { + "recommended_denies": int(raw_interventions.get("recommended_denies", 0)), + "applied_denies": int(raw_interventions.get("applied_denies", 0)), + "reviewed": int(raw_interventions.get("reviewed", 0)), + "false_stops": int(raw_interventions.get("false_stops", 0)), + } + governance = { + "tokens": int(raw_governance.get("external_tokens", 0)), + "usd": float(raw_governance.get("external_usd", 0.0)), + "latency_ms": float(raw_governance.get("total_latency_ms", 0.0)), + } + record: dict[str, Any] = { + "schema_version": 1, + "instance_id": config.instance_id, + "condition": config.condition, + "repetition": config.repetition, + "run_status": "completed", + "resolved": None, + "configuration_sha256": _configuration_hash(config), + "patch_sha256": hashlib.sha256(patch).hexdigest(), + "tokens": metrics.tokens, + "wall_time_ms": wall_time_ms, + "tool_calls": metrics.tool_calls, + "shell_commands": metrics.shell_commands, + "file_operations": metrics.file_operations, + "searches": metrics.searches, + "test_executions": metrics.test_executions, + "repeated_calls": metrics.repeated_calls, + "files_modified": files_modified, + "diff_lines": diff_lines, + "model_reroutes": None, + "interventions": interventions, + "governance": governance, + "error_code": None, + } + (run_dir / "run-record.json").write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (run_dir / "run-provenance.json").write_text( + json.dumps( + { + "base_commit": config.expected_base_commit, + "source_commit": config.source_commit, + "task_image": config.task_image, + "overlay_image": config.overlay_image, + "prompt_sha256": hashlib.sha256(config.prompt.encode()).hexdigest(), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return record diff --git a/benchmark/codex_adapter/container_runtime.py b/benchmark/codex_adapter/container_runtime.py new file mode 100644 index 0000000..543687e --- /dev/null +++ b/benchmark/codex_adapter/container_runtime.py @@ -0,0 +1,159 @@ +"""Fail-closed Docker boundary for official SWE-bench task images.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +_COMMIT = re.compile(r"^[0-9a-f]{40}$") +_DIGEST_IMAGE = re.compile(r"^[a-z0-9][a-z0-9._/-]*(?::[A-Za-z0-9._-]+)?@sha256:[0-9a-f]{64}$") +_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$") +_INSTANCE_ID = re.compile(r"^[A-Za-z0-9_.-]+__[A-Za-z0-9_.-]+-[0-9]+$") +_CONTAINER_NAME = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,127}$") + + +def official_instance_image(instance_id: str) -> str: + """Return the official SWE-bench Docker Hub tag for an instance ID.""" + + if _INSTANCE_ID.fullmatch(instance_id) is None: + raise ValueError("instance_id is not a valid SWE-bench instance ID") + escaped = instance_id.lower().replace("__", "_1776_") + return f"swebench/sweb.eval.x86_64.{escaped}:latest" + + +def _resolved_directory(path: Path, field: str) -> Path: + resolved = path.resolve() + if not resolved.is_dir(): + raise ValueError(f"{field} must be an existing directory") + if "," in str(resolved): + raise ValueError(f"{field} cannot contain a comma") + return resolved + + +def _resolved_file(path: Path, field: str) -> Path: + resolved = path.resolve() + if not resolved.is_file(): + raise ValueError(f"{field} must be an existing file") + if "," in str(resolved): + raise ValueError(f"{field} cannot contain a comma") + return resolved + + +@dataclass(frozen=True, slots=True) +class ContainerRunConfig: + """All public, non-secret inputs required to launch one disposable task container.""" + + instance_id: str + condition: str + expected_base_commit: str + task_image: str + overlay_image: str + container_name: str + run_dir: Path + source_root: Path + auth_source: Path + prompt_file: Path + model: str = "gpt-5.6-sol" + reasoning_effort: str = "high" + timeout_seconds: int = 1800 + memory: str = "12g" + cpus: str = "6" + + def __post_init__(self) -> None: + if _INSTANCE_ID.fullmatch(self.instance_id) is None: + raise ValueError("instance_id is not a valid SWE-bench instance ID") + if self.condition not in {"baseline", "marginal"}: + raise ValueError("condition must be baseline or marginal") + if _COMMIT.fullmatch(self.expected_base_commit) is None: + raise ValueError("expected_base_commit must be a lowercase 40-character SHA") + if _DIGEST_IMAGE.fullmatch(self.task_image) is None: + raise ValueError("task_image must be digest-pinned") + if ( + _DIGEST_IMAGE.fullmatch(self.overlay_image) is None + and _IMAGE_ID.fullmatch(self.overlay_image) is None + ): + raise ValueError("overlay_image must be digest-pinned") + if _CONTAINER_NAME.fullmatch(self.container_name) is None: + raise ValueError("container_name is invalid") + if not self.model.strip(): + raise ValueError("model must not be empty") + if not self.reasoning_effort.strip(): + raise ValueError("reasoning_effort must not be empty") + if isinstance(self.timeout_seconds, bool) or self.timeout_seconds < 1: + raise ValueError("timeout_seconds must be positive") + + source_root = _resolved_directory(self.source_root, "source_root") + run_dir = _resolved_directory(self.run_dir, "run_dir") + prompt_file = _resolved_file(self.prompt_file, "prompt_file") + _resolved_file(self.auth_source, "auth_source") + if run_dir == source_root or run_dir.is_relative_to(source_root): + raise ValueError("run_dir must be outside source_root") + if prompt_file.parent != run_dir: + raise ValueError("prompt_file must be directly inside run_dir") + + +def _bind(source: Path, destination: str, *, readonly: bool = False) -> str: + options = f",dst={destination}" + if readonly: + options += ",readonly" + return f"type=bind,src={source.resolve()}{options}" + + +def build_container_command(config: ContainerRunConfig) -> list[str]: + """Build a shell-free Docker command with a symmetric OFF/ON runtime boundary.""" + + return [ + "docker", + "run", + "--rm", + "--init", + "--platform", + "linux/amd64", + "--name", + config.container_name, + "--label", + f"org.marginal.benchmark.instance={config.instance_id}", + "--label", + f"org.marginal.benchmark.condition={config.condition}", + "--cap-add=SYS_ADMIN", + "--security-opt", + "seccomp=unconfined", + "--security-opt", + "apparmor=unconfined", + "--security-opt", + "systempaths=unconfined", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "2048", + "--memory", + config.memory, + "--cpus", + config.cpus, + "--network", + "bridge", + "--tmpfs", + "/marginal-home:rw,noexec,nosuid,nodev,size=64m", + "--tmpfs", + "/tmp:rw,exec,nosuid,nodev,size=2g", + "--mount", + _bind(config.run_dir, "/marginal-run"), + "--mount", + _bind(config.auth_source, "/run/secrets/codex-auth.json", readonly=True), + "--env", + f"MARGINAL_INSTANCE_ID={config.instance_id}", + "--env", + f"MARGINAL_CONDITION={config.condition}", + "--env", + f"MARGINAL_EXPECTED_BASE_COMMIT={config.expected_base_commit}", + "--env", + f"MARGINAL_MODEL={config.model}", + "--env", + f"MARGINAL_REASONING_EFFORT={config.reasoning_effort}", + "--env", + f"MARGINAL_TIMEOUT_SECONDS={config.timeout_seconds}", + "--entrypoint", + "/opt/marginal/benchmark/container/entrypoint.sh", + config.overlay_image, + ] diff --git a/benchmark/codex_adapter/daemon.py b/benchmark/codex_adapter/daemon.py new file mode 100644 index 0000000..996408b --- /dev/null +++ b/benchmark/codex_adapter/daemon.py @@ -0,0 +1,124 @@ +"""JSON-over-Unix-socket daemon that persists MARGINAL state for one task.""" + +from __future__ import annotations + +import argparse +import json +import signal +import socketserver +import threading +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .engine import CodexGovernanceEngine, IntegrationError + +_MAX_REQUEST_BYTES = 16 * 1024 * 1024 + + +class GovernanceUnixServer(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + """Threaded local server with one shared task-scoped governance engine.""" + + daemon_threads = True + + def __init__( + self, + server_address: str, + handler_class: type[socketserver.StreamRequestHandler], + *, + engine: CodexGovernanceEngine, + ) -> None: + self.engine = engine + super().__init__(server_address, handler_class) + + +class GovernanceRequestHandler(socketserver.StreamRequestHandler): + """Handle exactly one bounded request per local socket connection.""" + + server: GovernanceUnixServer + + def handle(self) -> None: + raw = self.rfile.readline(_MAX_REQUEST_BYTES + 1) + if len(raw) > _MAX_REQUEST_BYTES: + self._write_error("INVALID_REQUEST", "request exceeds size limit") + return + try: + request = json.loads(raw) + if not isinstance(request, Mapping): + raise TypeError("request must be a JSON object") + operation = request.get("operation") + payload = request.get("payload", {}) + if not isinstance(operation, str) or not operation: + raise ValueError("operation must be a non-empty string") + if not isinstance(payload, Mapping): + raise TypeError("payload must be a JSON object") + result = self._dispatch(operation, payload) + except IntegrationError as exc: + self._write_error("INTEGRATION_ERROR", str(exc)) + return + except (json.JSONDecodeError, TypeError, ValueError) as exc: + self._write_error("INVALID_REQUEST", str(exc)) + return + except Exception as exc: # pragma: no cover - last-resort protocol containment + self._write_error("INTERNAL_ERROR", f"{type(exc).__name__}: {exc}") + return + self._write({"ok": True, "result": result}) + + def _dispatch(self, operation: str, payload: Mapping[str, Any]) -> dict[str, Any]: + if operation == "health": + return {"status": "ready"} + if operation == "pre_tool_use": + return self.server.engine.pre_tool_use(payload) + if operation == "post_tool_use": + return self.server.engine.post_tool_use(payload) + if operation == "summary": + return self.server.engine.summary() + raise ValueError(f"unknown operation: {operation}") + + def _write_error(self, code: str, message: str) -> None: + self._write({"ok": False, "error_code": code, "message": message}) + + def _write(self, payload: Mapping[str, Any]) -> None: + self.wfile.write(json.dumps(payload, sort_keys=True).encode("utf-8") + b"\n") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--socket", required=True, type=Path) + parser.add_argument("--events", required=True, type=Path) + parser.add_argument("--summary", required=True, type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + args.socket.parent.mkdir(parents=True, exist_ok=True) + args.events.parent.mkdir(parents=True, exist_ok=True) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.socket.unlink(missing_ok=True) + + engine = CodexGovernanceEngine(events_path=args.events) + server = GovernanceUnixServer(str(args.socket), GovernanceRequestHandler, engine=engine) + stopping = threading.Event() + + def stop_server(_signum: int, _frame: object) -> None: + if not stopping.is_set(): + stopping.set() + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, stop_server) + signal.signal(signal.SIGINT, stop_server) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + args.summary.write_text( + json.dumps(engine.summary(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + args.socket.unlink(missing_ok=True) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/benchmark/codex_adapter/dataset.py b/benchmark/codex_adapter/dataset.py new file mode 100644 index 0000000..0c08da0 --- /dev/null +++ b/benchmark/codex_adapter/dataset.py @@ -0,0 +1,215 @@ +"""Frozen SWE-bench Lite task acquisition without exposing gold artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_DATASET = "princeton-nlp/SWE-bench_Lite" +_CONFIG = "default" +_SPLIT = "dev" +_VIEWER = "https://datasets-server.huggingface.co" +_HEX_40 = re.compile(r"^[0-9a-f]{40}$") +_HEX_64 = re.compile(r"^[0-9a-f]{64}$") +_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + + +class DatasetError(ValueError): + """Raised when live viewer data no longer matches the preregistered task manifest.""" + + +@dataclass(frozen=True, slots=True) +class FrozenTask: + instance_id: str + repo: str + base_commit: str + problem_statement: str + hints_text: str + + +def _sha(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _text(mapping: dict[str, Any], field: str) -> str: + value = mapping.get(field) + if not isinstance(value, str): + raise DatasetError(f"{field} must be a string") + return value + + +def task_specs_from_viewer( + manifest: dict[str, Any], viewer_rows: list[dict[str, Any]] +) -> tuple[FrozenTask, ...]: + """Build an allowlisted task tuple after matching every frozen identity and hash.""" + + if manifest.get("dataset") != _DATASET or manifest.get("split") != _SPLIT: + raise DatasetError("dataset and split must match the frozen SWE-bench Lite dev set") + task_ids = manifest.get("task_ids") + details = manifest.get("task_details") + if not isinstance(task_ids, list) or not all( + isinstance(item, str) and item for item in task_ids + ): + raise DatasetError("task_ids must contain non-empty strings") + if len(task_ids) != len(set(task_ids)): + raise DatasetError("task_ids contains duplicate IDs") + if not isinstance(details, list) or not all(isinstance(item, dict) for item in details): + raise DatasetError("task_details must contain objects") + detail_ids = [item.get("instance_id") for item in details] + if detail_ids != task_ids: + raise DatasetError("task_details order must exactly match task_ids") + + indexed_rows: dict[str, dict[str, Any]] = {} + for row in viewer_rows: + instance_id = row.get("instance_id") + if isinstance(instance_id, str) and instance_id in task_ids: + if instance_id in indexed_rows: + raise DatasetError(f"duplicate viewer row: {instance_id}") + indexed_rows[instance_id] = row + if set(indexed_rows) != set(task_ids): + missing = sorted(set(task_ids) - set(indexed_rows)) + raise DatasetError(f"viewer rows are missing frozen tasks: {missing}") + + tasks: list[FrozenTask] = [] + for detail in details: + instance_id = _text(detail, "instance_id") + row = indexed_rows[instance_id] + repo = _text(detail, "repo") + base_commit = _text(detail, "base_commit") + problem_hash = _text(detail, "problem_statement_sha256") + hints_hash = _text(detail, "hints_text_sha256") + if _REPO.fullmatch(repo) is None: + raise DatasetError(f"invalid repo for {instance_id}") + if _HEX_40.fullmatch(base_commit) is None: + raise DatasetError(f"invalid base_commit for {instance_id}") + if _HEX_64.fullmatch(problem_hash) is None or _HEX_64.fullmatch(hints_hash) is None: + raise DatasetError(f"invalid frozen text hash for {instance_id}") + if row.get("repo") != repo or row.get("base_commit") != base_commit: + raise DatasetError(f"repository identity changed for {instance_id}") + problem = _text(row, "problem_statement") + hints = _text(row, "hints_text") + if _sha(problem) != problem_hash: + raise DatasetError(f"problem_statement_sha256 mismatch for {instance_id}") + if _sha(hints) != hints_hash: + raise DatasetError(f"hints_text_sha256 mismatch for {instance_id}") + tasks.append(FrozenTask(instance_id, repo, base_commit, problem, hints)) + return tuple(tasks) + + +def _viewer_json(endpoint: str, **parameters: str | int) -> Any: + query = urllib.parse.urlencode(parameters) + request = urllib.request.Request( + f"{_VIEWER}/{endpoint}?{query}", + headers={"User-Agent": "marginal-codex-benchmark/1"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read()) + except (OSError, json.JSONDecodeError) as exc: + raise DatasetError(f"Hugging Face Dataset Viewer {endpoint} request failed") from exc + + +def fetch_frozen_tasks(manifest_path: str | Path) -> tuple[FrozenTask, ...]: + """Fetch public dev rows through Dataset Viewer and verify the frozen allowlist.""" + + path = Path(manifest_path) + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise DatasetError(f"invalid task manifest: {path}") from exc + if not isinstance(manifest, dict): + raise DatasetError("task manifest must be a JSON object") + + validity = _viewer_json("is-valid", dataset=_DATASET) + if not isinstance(validity, dict) or validity.get("viewer") is not True: + raise DatasetError("SWE-bench Lite Dataset Viewer is unavailable") + split_data = _viewer_json("splits", dataset=_DATASET) + splits = split_data.get("splits") if isinstance(split_data, dict) else None + if not isinstance(splits, list) or not any( + isinstance(item, dict) and item.get("config") == _CONFIG and item.get("split") == _SPLIT + for item in splits + ): + raise DatasetError("frozen default/dev split is unavailable") + row_data = _viewer_json( + "rows", + dataset=_DATASET, + config=_CONFIG, + split=_SPLIT, + offset=0, + length=100, + ) + wrapped_rows = row_data.get("rows") if isinstance(row_data, dict) else None + if not isinstance(wrapped_rows, list): + raise DatasetError("Dataset Viewer rows response is malformed") + rows: list[dict[str, Any]] = [] + for item in wrapped_rows: + row = item.get("row") if isinstance(item, dict) else None + if not isinstance(row, dict): + raise DatasetError("Dataset Viewer returned a malformed row") + rows.append(row) + return task_specs_from_viewer(manifest, rows) + + +def render_prompt(template_path: str | Path, problem_statement: str) -> str: + template = Path(template_path).read_text(encoding="utf-8") + marker = "{{problem_statement}}" + if template.count(marker) != 1: + raise DatasetError("prompt template must contain exactly one problem marker") + if not isinstance(problem_statement, str) or not problem_statement.strip(): + raise DatasetError("problem_statement must not be empty") + return template.replace(marker, problem_statement) + + +def _run(command: list[str], *, cwd: Path) -> str: + completed = subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + timeout=300, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() + raise DatasetError(f"command failed ({' '.join(command)}): {detail}") + return completed.stdout.strip() + + +def materialize_task( + task: FrozenTask, + destination: str | Path, + *, + repository_url: str | None = None, +) -> Path: + """Create a detached depth-one checkout containing only the frozen base commit.""" + + target = Path(destination).resolve() + if target.exists(): + raise DatasetError(f"destination already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + target.mkdir() + url = repository_url or f"https://github.com/{task.repo}.git" + _run(["git", "init", "-q"], cwd=target) + _run(["git", "remote", "add", "origin", url], cwd=target) + _run( + ["git", "fetch", "--depth", "1", "--filter=blob:none", "origin", task.base_commit], + cwd=target, + ) + _run(["git", "checkout", "--detach", "-q", "FETCH_HEAD"], cwd=target) + actual = _run(["git", "rev-parse", "HEAD"], cwd=target) + if actual != task.base_commit: + raise DatasetError(f"materialized commit mismatch: {actual}") + _run(["git", "remote", "remove", "origin"], cwd=target) + reachable = _run(["git", "rev-list", "--all", "--count"], cwd=target) + if reachable != "1": + raise DatasetError("materialized checkout exposes commits beyond the frozen base") + if _run(["git", "status", "--porcelain"], cwd=target): + raise DatasetError("materialized checkout is not clean") + return target diff --git a/benchmark/codex_adapter/engine.py b/benchmark/codex_adapter/engine.py new file mode 100644 index 0000000..835911f --- /dev/null +++ b/benchmark/codex_adapter/engine.py @@ -0,0 +1,204 @@ +"""Persistent MARGINAL governance state for one Codex benchmark task.""" + +from __future__ import annotations + +import hashlib +import json +import threading +import time +from collections.abc import Callable, Mapping +from dataclasses import replace +from pathlib import Path +from typing import Any + +from marginal.budget import BudgetLimits +from marginal.controls import DiminishingReturnDetector +from marginal.policy import MarginalPolicy +from marginal.profiles import PolicyProfile, policy_config_for_profile +from marginal.trace import JsonlTraceSink +from marginal.treasury import Treasury + +from .normalization import normalize_pre_tool_use +from .workspace import workspace_state_hash + + +class IntegrationError(RuntimeError): + """Raised when hook lifecycle state is inconsistent or cannot be settled safely.""" + + +StateHasher = Callable[[str | Path], str] + + +def _canonical_hash(value: object) -> str: + try: + encoded = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise IntegrationError("tool_response is not canonical JSON") from exc + return hashlib.sha256(encoded).hexdigest() + + +def _required_text(payload: Mapping[str, Any], field: str) -> str: + value = payload.get(field) + if not isinstance(value, str): + raise TypeError(f"{field} must be a string") + if not value.strip(): + raise ValueError(f"{field} must not be empty") + return value + + +class CodexGovernanceEngine: + """Own one in-memory Treasury across every tool call in a Codex task.""" + + def __init__( + self, + *, + events_path: str | Path, + state_hasher: StateHasher = workspace_state_hash, + ) -> None: + detector = DiminishingReturnDetector() + policy = MarginalPolicy( + policy_config_for_profile(PolicyProfile.BALANCED), + name="profile:balanced", + version="2.0.0", + diminishing_detector=detector, + ) + self._events = JsonlTraceSink(events_path) + self._treasury = Treasury( + BudgetLimits(), + policy=policy, + trace_sink=self._events, + mode="enforce", + name="codex-task", + ) + self._state_hasher = state_hasher + self._pending: dict[str, Any] = {} + self._evidence_by_semantic_key: dict[str, str] = {} + self._applied_denies = 0 + self._lock = threading.RLock() + + def pre_tool_use(self, payload: Mapping[str, Any]) -> dict[str, Any]: + """Authorize a proposed Codex tool call and reserve it when allowed.""" + + if not isinstance(payload, Mapping): + raise TypeError("payload must be a mapping") + tool_use_id = _required_text(payload, "tool_use_id") + cwd = _required_text(payload, "cwd") + with self._lock: + if tool_use_id in self._pending: + raise IntegrationError(f"tool_use_id {tool_use_id!r} is already pending") + + started = time.perf_counter_ns() + state_hash = self._state_hasher(cwd) + proposal = normalize_pre_tool_use(payload, state_hash=state_hash) + semantic_key = str(proposal.action.metadata["marginal_semantic_key"]) + previous_evidence = self._evidence_by_semantic_key.get(semantic_key, "") + if previous_evidence: + proposal = normalize_pre_tool_use( + payload, + state_hash=state_hash, + previous_evidence_hash=previous_evidence, + ) + preparation_ms = (time.perf_counter_ns() - started) / 1_000_000 + self._treasury.record_governance_overhead(latency_ms=preparation_ms) + + diminishing = self._treasury.policy.diminishing_signal(proposal.action) + decision = self._treasury.authorize(proposal.action) + if decision.allowed: + self._pending[tool_use_id] = proposal.action + else: + self._applied_denies += 1 + + result = { + "allowed": decision.allowed, + "recommended": decision.recommended, + "reason": decision.reason, + "reason_code": decision.reason_code, + "expected_gain": decision.expected_gain, + "same_state_repeats": ( + diminishing.same_state_repeats if diminishing is not None else 0 + ), + } + self._events.emit( + { + "event": "codex_pre_tool_use", + "tool_use_id": tool_use_id, + "tool_name": payload.get("tool_name"), + **result, + } + ) + return result + + def post_tool_use(self, payload: Mapping[str, Any]) -> dict[str, Any]: + """Settle a previously authorized tool call with observed state and evidence.""" + + if not isinstance(payload, Mapping): + raise TypeError("payload must be a mapping") + tool_use_id = _required_text(payload, "tool_use_id") + cwd = _required_text(payload, "cwd") + if "tool_response" not in payload: + raise ValueError("tool_response is required") + with self._lock: + action = self._pending.get(tool_use_id) + if action is None: + raise IntegrationError(f"tool_use_id {tool_use_id!r} is not pending") + + started = time.perf_counter_ns() + state_hash = self._state_hasher(cwd) + post_proposal = normalize_pre_tool_use(payload, state_hash=state_hash) + identity_fields = ("session_id", "turn_id", "tool_name", "cwd") + identity_matches = all( + post_proposal.action.metadata[field] == action.metadata[field] + for field in identity_fields + ) and ( + post_proposal.action.metadata["marginal_semantic_key"] + == action.metadata["marginal_semantic_key"] + ) + if not identity_matches: + raise IntegrationError(f"PostToolUse identity does not match {tool_use_id!r}") + evidence_hash = _canonical_hash(payload["tool_response"]) + metadata = dict(action.metadata) + metadata.update({"state_hash": state_hash, "evidence_hash": evidence_hash}) + settled_action = replace(action, metadata=metadata) + preparation_ms = (time.perf_counter_ns() - started) / 1_000_000 + self._treasury.record_governance_overhead(latency_ms=preparation_ms) + + # PostToolUse is Codex's authoritative signal that the tool handler + # completed. An application-level non-zero exit (for example, a red test) + # is still evidence-producing work. Codex 0.147 does not expose that exit + # status here, so classifying it as a transport failure would erase the + # exact repeated verification behavior this adapter is designed to govern. + self._treasury.commit(settled_action) + del self._pending[tool_use_id] + semantic_key = str(settled_action.metadata["marginal_semantic_key"]) + self._evidence_by_semantic_key[semantic_key] = evidence_hash + self._events.emit( + { + "event": "codex_post_tool_use", + "tool_use_id": tool_use_id, + "tool_name": payload.get("tool_name"), + "state_hash": state_hash, + "evidence_hash": evidence_hash, + "completed": True, + } + ) + return {"settled": True, "completed": True} + + def summary(self) -> dict[str, Any]: + with self._lock: + summary = self._treasury.summary() + governance = summary["governance"] + assert isinstance(governance, dict) + summary["pending"] = len(self._pending) + summary["interventions"] = { + "recommended_denies": self._applied_denies, + "applied_denies": self._applied_denies, + "reviewed": governance["reviewed_stops"], + "false_stops": governance["false_stops"], + } + return summary diff --git a/benchmark/codex_adapter/evidence.py b/benchmark/codex_adapter/evidence.py new file mode 100644 index 0000000..8afd85f --- /dev/null +++ b/benchmark/codex_adapter/evidence.py @@ -0,0 +1,186 @@ +"""Export attested private Codex runs into verifier-safe SWE-bench evidence.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +class EvidenceError(ValueError): + """Raised when private run artifacts cannot support a public comparison.""" + + +@dataclass(frozen=True, slots=True) +class EvidenceExportConfig: + raw_root: Path + output_dir: Path + task_set: str + instance_ids: tuple[str, ...] + task_set_sha256: str + prompt_template_sha256: str + source_commit: str + model: str = "gpt-5.6-sol" + agent_version: str = "0.147.0" + timeout_seconds: int = 1800 + + def __post_init__(self) -> None: + if self.task_set not in {"smoke", "canary"}: + raise ValueError("task_set must be smoke or canary") + if not self.instance_ids or len(self.instance_ids) != len(set(self.instance_ids)): + raise ValueError("instance_ids must be non-empty and unique") + if re.fullmatch(r"[0-9a-f]{64}", self.task_set_sha256) is None: + raise ValueError("task_set_sha256 must be a SHA-256 digest") + if re.fullmatch(r"[0-9a-f]{64}", self.prompt_template_sha256) is None: + raise ValueError("prompt_template_sha256 must be a SHA-256 digest") + if re.fullmatch(r"[0-9a-f]{40}", self.source_commit) is None: + raise ValueError("source_commit must be a commit SHA") + + +def _object(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise EvidenceError(f"invalid or missing artifact: {path}") from exc + if not isinstance(value, dict): + raise EvidenceError(f"artifact must contain an object: {path}") + return value + + +def _write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text( + "".join( + json.dumps(row, allow_nan=False, sort_keys=True, separators=(",", ":")) + "\n" + for row in rows + ), + encoding="utf-8", + ) + + +def _validated_run( + config: EvidenceExportConfig, + instance_id: str, + condition: str, +) -> tuple[dict[str, Any], dict[str, Any], str]: + run_dir = config.raw_root / instance_id / condition + record = _object(run_dir / "run-record.json") + provenance = _object(run_dir / "run-provenance.json") + if record.get("instance_id") != instance_id or record.get("condition") != condition: + raise EvidenceError(f"run identity mismatch: {instance_id}/{condition}") + if record.get("run_status") != "completed" or record.get("error_code") is not None: + raise EvidenceError(f"run is not completed: {instance_id}/{condition}") + if record.get("resolved") is not None: + raise EvidenceError("resolved must remain verifier-owned before export") + if provenance.get("source_commit") != config.source_commit: + raise EvidenceError(f"source commit mismatch: {instance_id}/{condition}") + patch_path = run_dir / "model.patch" + try: + patch_bytes = patch_path.read_bytes() + patch = patch_bytes.decode("utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise EvidenceError(f"model patch is unavailable or non-UTF-8: {patch_path}") from exc + if hashlib.sha256(patch_bytes).hexdigest() != record.get("patch_sha256"): + raise EvidenceError(f"patch digest mismatch: {instance_id}/{condition}") + tokens = record.get("tokens") + if not isinstance(tokens, dict) or not isinstance(tokens.get("total"), int): + raise EvidenceError(f"token telemetry missing: {instance_id}/{condition}") + return record, provenance, patch + + +def export_evidence(config: EvidenceExportConfig) -> Path: + """Export matched predictions/metrics and public provenance in frozen order.""" + + output = config.output_dir.resolve() + if output.exists(): + raise EvidenceError(f"output directory already exists: {output}") + output.mkdir(parents=True) + predictions: dict[str, list[dict[str, Any]]] = {"baseline": [], "marginal": []} + metrics: dict[str, list[dict[str, Any]]] = {"baseline": [], "marginal": []} + public_records: list[dict[str, Any]] = [] + task_provenance: list[dict[str, Any]] = [] + + for instance_id in config.instance_ids: + lane_provenance: dict[str, dict[str, Any]] = {} + for condition in ("baseline", "marginal"): + record, provenance, patch = _validated_run(config, instance_id, condition) + lane_provenance[condition] = provenance + governance = record.get("governance") + interventions = record.get("interventions") + if not isinstance(governance, dict) or not isinstance(interventions, dict): + raise EvidenceError(f"accounting objects missing: {instance_id}/{condition}") + predictions[condition].append( + { + "instance_id": instance_id, + "model_name_or_path": config.model, + "model_patch": patch, + } + ) + metrics[condition].append( + { + "instance_id": instance_id, + "tokens": int(record["tokens"]["total"]), + "latency_ms": int(record["wall_time_ms"]), + "tool_calls": int(record["tool_calls"]), + "repeated_calls": int(record["repeated_calls"]), + "usd": 0.0, + "governance_tokens": int(governance.get("tokens", 0)), + "governance_usd": float(governance.get("usd", 0.0)), + "governance_latency_ms": round(float(governance.get("latency_ms", 0.0))), + "reviewed_stops": int(interventions.get("reviewed", 0)), + "false_stops": int(interventions.get("false_stops", 0)), + "applied_denies": int(interventions.get("applied_denies", 0)), + } + ) + public_records.append(record) + baseline_provenance = lane_provenance["baseline"] + marginal_provenance = lane_provenance["marginal"] + for field in ("base_commit", "task_image", "prompt_sha256"): + if baseline_provenance.get(field) != marginal_provenance.get(field): + raise EvidenceError(f"lane provenance mismatch for {instance_id}: {field}") + task_provenance.append( + { + "instance_id": instance_id, + "base_commit": baseline_provenance["base_commit"], + "task_image": baseline_provenance["task_image"], + "baseline_overlay_image": baseline_provenance["overlay_image"], + "marginal_overlay_image": marginal_provenance["overlay_image"], + "prompt_sha256": baseline_provenance["prompt_sha256"], + } + ) + + manifest = { + "schema_version": 1, + "benchmark": "swe-bench-lite-dev-canary", + "dataset": "princeton-nlp/SWE-bench_Lite", + "split": "dev", + "task_set": config.task_set, + "task_set_sha256": config.task_set_sha256, + "agent": "codex-cli", + "agent_version": config.agent_version, + "model": config.model, + "prompt_sha256": config.prompt_template_sha256, + "limits": {"timeout_seconds": config.timeout_seconds, "repetitions": 1}, + "marginal": { + "version": "0.2.0+unreleased", + "commit": config.source_commit, + "mode": "enforce", + "policy": "balanced+diminishing-defaults", + }, + } + _write_json(output / "manifest.json", manifest) + for condition in ("baseline", "marginal"): + _write_jsonl(output / f"{condition}_predictions.ndjson", predictions[condition]) + _write_jsonl(output / f"{condition}_metrics.ndjson", metrics[condition]) + _write_jsonl(output / "run-records.ndjson", public_records) + _write_json(output / "provenance.json", {"tasks": task_provenance}) + return output diff --git a/benchmark/codex_adapter/hook_client.py b/benchmark/codex_adapter/hook_client.py new file mode 100644 index 0000000..569dfe6 --- /dev/null +++ b/benchmark/codex_adapter/hook_client.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Thin Codex hook process that forwards stdin to the task-scoped daemon.""" + +from __future__ import annotations + +import argparse +import json +import os +import socket +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any, TextIO + +_MAX_MESSAGE_BYTES = 16 * 1024 * 1024 + + +def hook_output(operation: str, result: Mapping[str, Any]) -> dict[str, Any] | None: + """Translate a daemon result to the official Codex command-hook output shape.""" + + if operation == "post": + if result.get("settled") is not True: + raise RuntimeError("PostToolUse was not settled") + return None + if operation != "pre": + raise ValueError(f"unsupported hook operation: {operation}") + allowed = result.get("allowed") + if not isinstance(allowed, bool): + raise RuntimeError("daemon pre result is missing boolean allowed") + if allowed: + return None + reason = result.get("reason") + if not isinstance(reason, str) or not reason.strip(): + raise RuntimeError("daemon deny result is missing a reason") + reason_code = result.get("reason_code") + suffix = f" [{reason_code}]" if isinstance(reason_code, str) and reason_code else "" + return { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": f"{reason}{suffix}", + } + } + + +def _request_daemon( + socket_path: Path, operation: str, payload: Mapping[str, Any] +) -> Mapping[str, Any]: + encoded = json.dumps( + {"operation": operation, "payload": payload}, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ).encode("utf-8") + if len(encoded) > _MAX_MESSAGE_BYTES: + raise RuntimeError("hook request exceeds size limit") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(60) + client.connect(str(socket_path)) + client.sendall(encoded + b"\n") + response = client.makefile("rb").readline(_MAX_MESSAGE_BYTES + 1) + if not response: + raise RuntimeError("daemon closed the connection without a response") + if len(response) > _MAX_MESSAGE_BYTES: + raise RuntimeError("daemon response exceeds size limit") + decoded = json.loads(response) + if not isinstance(decoded, Mapping): + raise RuntimeError("daemon response must be a JSON object") + if decoded.get("ok") is not True: + code = decoded.get("error_code", "UNKNOWN") + message = decoded.get("message", "daemon rejected request") + raise RuntimeError(f"daemon {code}: {message}") + result = decoded.get("result") + if not isinstance(result, Mapping): + raise RuntimeError("daemon response is missing an object result") + return result + + +def run_hook( + operation: str, + *, + stdin: TextIO, + stdout: TextIO, + stderr: TextIO, + socket_path: Path, + failure_log: Path | None = None, +) -> int: + """Run one synchronous hook invocation with explicit integration failure output.""" + + try: + raw = stdin.read(_MAX_MESSAGE_BYTES + 1) + if len(raw.encode("utf-8")) > _MAX_MESSAGE_BYTES: + raise RuntimeError("hook stdin exceeds size limit") + payload = json.loads(raw) + if not isinstance(payload, Mapping): + raise TypeError("hook stdin must be a JSON object") + daemon_operation = {"pre": "pre_tool_use", "post": "post_tool_use"}[operation] + result = _request_daemon(socket_path, daemon_operation, payload) + output = hook_output(operation, result) + if output is not None: + stdout.write(json.dumps(output, sort_keys=True) + "\n") + return 0 + except Exception as exc: + message = f"MARGINAL integration failure: {type(exc).__name__}: {exc}" + stderr.write(message + "\n") + if failure_log is not None: + failure_log.parent.mkdir(parents=True, exist_ok=True) + with failure_log.open("a", encoding="utf-8") as stream: + stream.write(message + "\n") + return 1 + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("pre", "post")) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + socket_value = os.environ.get("MARGINAL_SOCKET", "") + if not socket_value: + sys.stderr.write("MARGINAL integration failure: MARGINAL_SOCKET is not set\n") + return 1 + failure_log_value = os.environ.get("MARGINAL_HOOK_FAILURE_LOG", "") + return run_hook( + args.operation, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + socket_path=Path(socket_value), + failure_log=Path(failure_log_value) if failure_log_value else None, + ) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/benchmark/codex_adapter/hook_config.py b/benchmark/codex_adapter/hook_config.py new file mode 100644 index 0000000..8c6867f --- /dev/null +++ b/benchmark/codex_adapter/hook_config.py @@ -0,0 +1,59 @@ +"""Install the project-local Codex hook declaration used only in the ON lane.""" + +from __future__ import annotations + +import json +import shlex +from pathlib import Path + + +def install_project_hooks( + worktree: str | Path, + *, + python_executable: str | Path, + hook_client: str | Path, + timeout_seconds: int = 60, +) -> Path: + """Write deterministic synchronous PreToolUse/PostToolUse command hooks.""" + + root = Path(worktree).resolve() + python_path = Path(python_executable).resolve() + client_path = Path(hook_client).resolve() + if isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, int): + raise TypeError("timeout_seconds must be an integer") + if timeout_seconds < 1: + raise ValueError("timeout_seconds must be positive") + if not root.is_dir(): + raise ValueError(f"worktree is not a directory: {root}") + if not python_path.is_file(): + raise ValueError(f"Python executable does not exist: {python_path}") + if not client_path.is_file(): + raise ValueError(f"hook client does not exist: {client_path}") + + def handler(operation: str) -> dict[str, object]: + command = shlex.join((str(python_path), str(client_path), operation)) + return { + "hooks": [ + { + "type": "command", + "command": command, + "timeout": timeout_seconds, + "async": False, + } + ] + } + + config = { + "description": "MARGINAL benchmark adapter; generated per ON-lane task.", + "hooks": { + "PreToolUse": [handler("pre")], + "PostToolUse": [handler("post")], + }, + } + config_path = root / ".codex" / "hooks.json" + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text( + json.dumps(config, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return config_path diff --git a/benchmark/codex_adapter/normalization.py b/benchmark/codex_adapter/normalization.py new file mode 100644 index 0000000..869132a --- /dev/null +++ b/benchmark/codex_adapter/normalization.py @@ -0,0 +1,125 @@ +"""Normalize Codex hook payloads into provider-neutral MARGINAL actions.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from marginal.models import Action, Cost + +_VERIFICATION_COMMAND = re.compile( + r"(?:^|[;&|]\s*)(?:\S*/)?(?:pytest|py\.test|tox|nox|go\s+test|cargo\s+test|" + r"npm\s+(?:run\s+)?test|pnpm\s+(?:run\s+)?test|yarn\s+(?:run\s+)?test|" + r"python(?:\d+(?:\.\d+)?)?\s+-m\s+(?:pytest|unittest))\b", + re.IGNORECASE, +) + + +@dataclass(frozen=True, slots=True) +class NormalizedProposal: + """Validated hook identity paired with the action MARGINAL evaluates.""" + + tool_use_id: str + action: Action + + +def _required_text(payload: Mapping[str, Any], field: str) -> str: + value = payload.get(field) + if not isinstance(value, str): + raise TypeError(f"{field} must be a string") + if not value.strip(): + raise ValueError(f"{field} must not be empty") + return value + + +def _canonical_json(value: Mapping[str, Any]) -> str: + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError) as exc: + raise ValueError("tool_input must contain canonical JSON values") from exc + + +def _classify(tool_name: str, tool_input: Mapping[str, Any]) -> tuple[str, bool]: + normalized = tool_name.strip().lower() + if normalized in {"bash", "shell", "shell_command", "exec_command"}: + command = tool_input.get("command", tool_input.get("cmd", "")) + if isinstance(command, str) and _VERIFICATION_COMMAND.search(command): + return "verification", True + return "shell", False + if normalized in {"apply_patch", "patch"}: + return "edit", False + if normalized.startswith("mcp__"): + return "mcp", False + return "tool", False + + +def _action_name(tool_name: str, tool_input: Mapping[str, Any]) -> str: + candidate = tool_input.get("command", tool_input.get("cmd", tool_input.get("path", ""))) + if not isinstance(candidate, str) or not candidate.strip(): + return tool_name + compact = " ".join(candidate.split()) + return f"{tool_name}: {compact[:160]}" + + +def normalize_pre_tool_use( + payload: Mapping[str, Any], + *, + state_hash: str, + previous_evidence_hash: str = "", +) -> NormalizedProposal: + """Validate one ``PreToolUse`` payload and build its MARGINAL action. + + Hook payloads do not expose a defensible forecast of future token or dollar cost, so + the benchmark deliberately reserves zero pre-execution cost. The resulting experiment + isolates state-aware repetition control rather than claiming economic ROI calibration. + """ + + if not isinstance(payload, Mapping): + raise TypeError("hook payload must be a mapping") + session_id = _required_text(payload, "session_id") + turn_id = _required_text(payload, "turn_id") + tool_use_id = _required_text(payload, "tool_use_id") + tool_name = _required_text(payload, "tool_name") + cwd = _required_text(payload, "cwd") + tool_input = payload.get("tool_input") + if not isinstance(tool_input, Mapping): + raise TypeError("tool_input must be a mapping") + if not isinstance(state_hash, str) or not state_hash.strip(): + raise ValueError("state_hash must not be empty") + if not isinstance(previous_evidence_hash, str): + raise TypeError("previous_evidence_hash must be a string") + + canonical_input = _canonical_json(tool_input) + semantic_material = f"{tool_name.strip().lower()}\0{canonical_input}".encode() + semantic_key = hashlib.sha256(semantic_material).hexdigest() + kind, is_verification = _classify(tool_name, tool_input) + action = Action( + name=_action_name(tool_name, tool_input), + kind=kind, + cost=Cost(), + expected_gain=None, + is_verification=is_verification, + fingerprint=f"codex:{tool_use_id}", + metadata={ + "phase": "codex-tool-use", + "session_id": session_id, + "turn_id": turn_id, + "tool_use_id": tool_use_id, + "tool_name": tool_name, + "cwd": cwd, + "state_hash": state_hash, + "evidence_hash": previous_evidence_hash, + "marginal_semantic_key": semantic_key, + }, + ) + return NormalizedProposal(tool_use_id=tool_use_id, action=action) diff --git a/benchmark/codex_adapter/preflight.py b/benchmark/codex_adapter/preflight.py new file mode 100644 index 0000000..2ebb3ae --- /dev/null +++ b/benchmark/codex_adapter/preflight.py @@ -0,0 +1,258 @@ +"""Fail-closed readiness checks performed before any Codex benchmark inference.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator # type: ignore[import-untyped] + +from .dataset import FrozenTask, fetch_frozen_tasks + + +class PreflightError(RuntimeError): + """Raised when execution would violate the frozen benchmark contract.""" + + +@dataclass(frozen=True, slots=True) +class PreflightConfig: + repository_root: Path + codex_executable: Path + python_executable: Path + auth_source: Path + environment_path: Path + tasks_path: Path + prompt_path: Path + schema_path: Path + expected_repository_commit: str + require_verifier: bool = True + require_task_environment: bool = True + + +TaskLoader = Callable[[str | Path], tuple[FrozenTask, ...]] + + +def _run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> str: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + timeout=60, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise PreflightError(f"command failed ({' '.join(command)}): {detail}") + return "\n".join(part for part in (completed.stdout.strip(), completed.stderr.strip()) if part) + + +def _json_object(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PreflightError(f"invalid JSON file: {path}") from exc + if not isinstance(value, dict): + raise PreflightError(f"JSON file must contain an object: {path}") + return value + + +def _nested_text(value: dict[str, Any], *keys: str) -> str: + current: Any = value + for key in keys: + if not isinstance(current, dict): + raise PreflightError(f"missing environment field: {'.'.join(keys)}") + current = current.get(key) + if not isinstance(current, str) or not current: + raise PreflightError(f"missing environment field: {'.'.join(keys)}") + return current + + +def run_preflight( + config: PreflightConfig, + *, + task_loader: TaskLoader = fetch_frozen_tasks, +) -> dict[str, Any]: + """Verify code, CLI, model, auth, schema, prompt, and task identities.""" + + root = config.repository_root.resolve() + for path, label in ( + (root, "repository"), + (config.codex_executable, "Codex executable"), + (config.python_executable, "Python executable"), + (config.auth_source, "Codex auth file"), + ): + if not path.exists(): + raise PreflightError(f"{label} does not exist: {path}") + + head = _run(["git", "rev-parse", "HEAD"], cwd=root) + if head != config.expected_repository_commit: + raise PreflightError(f"MARGINAL commit mismatch: {head}") + core_status = _run( + [ + "git", + "status", + "--porcelain", + "--untracked-files=no", + "--", + "src/marginal", + "pyproject.toml", + ], + cwd=root, + ) + if core_status: + raise PreflightError("MARGINAL core source is dirty") + + environment = _json_object(config.environment_path) + if _nested_text(environment, "repository", "commit") != head: + raise PreflightError("environment repository commit does not match HEAD") + expected_python = _nested_text(environment, "python") + python_output = _run([str(config.python_executable), "--version"], cwd=root) + actual_python = python_output.removeprefix("Python ").strip() + if actual_python != expected_python: + raise PreflightError(f"Python version mismatch: {actual_python}") + + expected_codex = _nested_text(environment, "codex", "cli_version") + version_output = _run([str(config.codex_executable), "--version"], cwd=root) + match = re.search(r"(\d+\.\d+\.\d+)", version_output) + actual_codex = match.group(1) if match else "" + if actual_codex != expected_codex: + raise PreflightError(f"Codex version mismatch: {actual_codex or version_output}") + features = _run([str(config.codex_executable), "features", "list"], cwd=root) + if re.search(r"(?m)^hooks\s+stable\s+true\s*$", features) is None: + raise PreflightError("Codex hooks feature is not stable and enabled") + + model = _nested_text(environment, "codex", "model") + reasoning_effort = _nested_text(environment, "codex", "reasoning_effort") + model_catalog_output = _run( + [str(config.codex_executable), "debug", "models", "--bundled"], cwd=root + ) + try: + catalog = json.loads(model_catalog_output) + except json.JSONDecodeError as exc: + raise PreflightError("Codex bundled model catalog is invalid") from exc + models = catalog.get("models") if isinstance(catalog, dict) else None + if not isinstance(models, list): + raise PreflightError("Codex bundled model catalog has no models list") + selected = next( + (entry for entry in models if isinstance(entry, dict) and entry.get("slug") == model), + None, + ) + if selected is None: + raise PreflightError(f"frozen model is unavailable: {model}") + levels = selected.get("supported_reasoning_levels") + if not isinstance(levels, list) or not any( + isinstance(level, dict) and level.get("effort") == reasoning_effort for level in levels + ): + raise PreflightError(f"model does not support reasoning effort: {reasoning_effort}") + + mode = stat.S_IMODE(config.auth_source.stat().st_mode) + if mode != 0o600: + raise PreflightError("Codex auth file permissions must be 0600") + with tempfile.TemporaryDirectory(prefix="codex-preflight-") as home_value: + temporary_home = Path(home_value) + auth_copy = temporary_home / "auth.json" + shutil.copyfile(config.auth_source, auth_copy) + auth_copy.chmod(0o600) + auth_environment = { + key: os.environ[key] + for key in ("HOME", "LANG", "LC_ALL", "PATH", "SHELL", "TMPDIR", "USER") + if key in os.environ + } + auth_environment["CODEX_HOME"] = str(temporary_home) + login_status = _run( + [str(config.codex_executable), "login", "status"], + cwd=root, + env=auth_environment, + ) + if "Logged in" not in login_status: + raise PreflightError("Codex authentication is not ready") + + prompt_hash = hashlib.sha256(config.prompt_path.read_bytes()).hexdigest() + frozen_prompt_hash = _nested_text(environment, "benchmark", "prompt_template_sha256") + if prompt_hash != frozen_prompt_hash: + raise PreflightError("prompt template hash mismatch") + tasks_manifest = _json_object(config.tasks_path) + task_ids = tasks_manifest.get("task_ids") + if not isinstance(task_ids, list) or not all(isinstance(item, str) for item in task_ids): + raise PreflightError("task manifest has invalid task_ids") + if len(task_ids) != len(set(task_ids)): + raise PreflightError("task manifest has duplicate task IDs") + loaded_tasks = task_loader(config.tasks_path) + if [task.instance_id for task in loaded_tasks] != task_ids: + raise PreflightError("Dataset Viewer task order does not match the frozen manifest") + task_digest = hashlib.sha256("\n".join(task_ids).encode("utf-8")).hexdigest() + declared_digest = tasks_manifest.get("task_set_sha256") + if declared_digest is not None and declared_digest != task_digest: + raise PreflightError("task-set digest mismatch") + + schema = _json_object(config.schema_path) + try: + Draft202012Validator.check_schema(schema) + except Exception as exc: + raise PreflightError("run-record JSON Schema is invalid") from exc + + if config.require_task_environment: + raise PreflightError( + "benchmark execution blocked: Codex 0.147 PostToolUse omits shell exit status, " + "so successful repeat detection is not observable; an official per-instance " + "Codex execution backend is also not implemented" + ) + + verifier = "not-required" + if config.require_verifier: + modal_credentials = bool(os.environ.get("MODAL_TOKEN_ID")) and bool( + os.environ.get("MODAL_TOKEN_SECRET") + ) + modal_packages = ( + importlib.util.find_spec("modal") is not None + and importlib.util.find_spec("swebench") is not None + ) + docker = shutil.which("docker") + docker_ready = False + if docker is not None and importlib.util.find_spec("swebench") is not None: + probe = subprocess.run( + [docker, "info"], + cwd=root, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) + docker_ready = probe.returncode == 0 + if modal_credentials and modal_packages: + verifier = "official-swebench-modal" + elif docker_ready: + verifier = "official-swebench-docker" + else: + raise PreflightError( + "official SWE-bench verifier unavailable: configure Modal credentials plus " + "swebench[modal], or provide a running Docker engine plus swebench" + ) + + return { + "ready": True, + "repository_commit": head, + "python_version": actual_python, + "codex_version": actual_codex, + "model": model, + "reasoning_effort": reasoning_effort, + "hooks": "stable-enabled", + "prompt_template_sha256": prompt_hash, + "task_set_sha256": task_digest, + "task_ids": task_ids, + "verifier": verifier, + "credentials_recorded": False, + } diff --git a/benchmark/codex_adapter/runner.py b/benchmark/codex_adapter/runner.py new file mode 100644 index 0000000..fa77f35 --- /dev/null +++ b/benchmark/codex_adapter/runner.py @@ -0,0 +1,605 @@ +"""Hermetic single-task runner for matched Codex OFF/ON conditions.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import time +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any + +from .codex_events import CodexMetrics, EventParseError, parse_codex_jsonl +from .hook_config import install_project_hooks + +_SOURCE_ROOT = Path(__file__).resolve().parents[2] +_HOOK_CLIENT = Path(__file__).resolve().with_name("hook_client.py") +_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +_FEATURES_DISABLED = ( + "apps", + "browser_use", + "browser_use_external", + "browser_use_full_cdp_access", + "computer_use", + "goals", + "image_generation", + "in_app_browser", + "multi_agent", + "multi_agent_v2", + "plugins", + "search_tool", + "skill_search", + "standalone_web_search", + "tool_suggest", + "web_search_request", + "workspace_dependencies", +) + + +@dataclass(frozen=True, slots=True) +class RunConfig: + instance_id: str + condition: str + repetition: int + worktree: Path + expected_base_commit: str + run_dir: Path + prompt: str + codex_executable: Path + auth_source: Path + model: str = "gpt-5.6-sol" + reasoning_effort: str = "high" + timeout_seconds: float = 1800 + codex_version: str = "0.147.0" + extra_env: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.condition not in {"baseline", "marginal"}: + raise ValueError("condition must be baseline or marginal") + if self.repetition < 1: + raise ValueError("repetition must be positive") + if _COMMIT_SHA.fullmatch(self.expected_base_commit) is None: + raise ValueError("expected_base_commit must be a lowercase 40-character SHA") + if self.timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + + +def _safe_environment(config: RunConfig, codex_home: Path, runtime_home: Path) -> dict[str, str]: + retained = ("LANG", "LC_ALL", "LOGNAME", "PATH", "SHELL", "TMPDIR", "USER") + environment = {key: os.environ[key] for key in retained if key in os.environ} + environment.update(config.extra_env) + environment.update( + { + "CODEX_HOME": str(codex_home), + "HOME": str(runtime_home), + "CLICOLOR": "0", + "TERM": "dumb", + } + ) + return environment + + +def _toml_inline_table(values: Mapping[str, str]) -> str: + fields = ",".join(f"{key}={json.dumps(value)}" for key, value in sorted(values.items())) + return "{" + fields + "}" + + +def _command(config: RunConfig, shell_environment: Mapping[str, str]) -> list[str]: + command = [ + str(config.codex_executable), + "exec", + "--ignore-user-config", + "--ignore-rules", + "--ephemeral", + "--json", + "--color", + "never", + "--strict-config", + "--dangerously-bypass-hook-trust", + "-m", + config.model, + "-c", + f'model_reasoning_effort="{config.reasoning_effort}"', + "-c", + "sandbox_workspace_write.network_access=false", + "-c", + 'shell_environment_policy.inherit="none"', + "-c", + "shell_environment_policy.ignore_default_excludes=false", + "-c", + f"shell_environment_policy.set={_toml_inline_table(shell_environment)}", + "-s", + "workspace-write", + "-a", + "never", + "-C", + str(config.worktree), + ] + for feature in _FEATURES_DISABLED: + command.extend(("--disable", feature)) + command.append("-") + return command + + +def _daemon_request(socket_path: Path, operation: str) -> dict[str, Any]: + request = json.dumps({"operation": operation, "payload": {}}).encode("utf-8") + b"\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(1) + client.connect(str(socket_path)) + client.sendall(request) + raw = client.makefile("rb").readline(16 * 1024 * 1024) + response = json.loads(raw) + if not isinstance(response, dict) or response.get("ok") is not True: + raise RuntimeError(f"daemon request failed: {response!r}") + result = response.get("result") + if not isinstance(result, dict): + raise RuntimeError("daemon response has no object result") + return result + + +def _wait_for_daemon(process: subprocess.Popen[bytes], socket_path: Path) -> None: + deadline = time.monotonic() + 10 + last_error: Exception | None = None + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"MARGINAL daemon exited with {process.returncode}") + if socket_path.exists(): + try: + if _daemon_request(socket_path, "health").get("status") == "ready": + return + except (OSError, RuntimeError, json.JSONDecodeError) as exc: + last_error = exc + time.sleep(0.02) + raise RuntimeError(f"MARGINAL daemon did not become ready: {last_error}") + + +def _git(worktree: Path, *args: str, allowed_codes: tuple[int, ...] = (0,)) -> bytes: + completed = subprocess.run( + ["git", *args], + cwd=worktree, + capture_output=True, + check=False, + timeout=60, + ) + if completed.returncode not in allowed_codes: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"git {' '.join(args)} failed: {detail}") + return completed.stdout + + +def _instrumentation_path(relative: str) -> bool: + return ".codex" in PurePosixPath(relative).parts + + +def _validate_starting_checkout(config: RunConfig, worktree: Path) -> None: + try: + head = _git(worktree, "rev-parse", "HEAD").decode("ascii").strip() + head_name = _git(worktree, "rev-parse", "--abbrev-ref", "HEAD").decode("ascii").strip() + status = _git(worktree, "status", "--porcelain", "--untracked-files=all") + except (UnicodeDecodeError, RuntimeError) as exc: + raise ValueError(f"task checkout validation failed: {exc}") from exc + if head != config.expected_base_commit: + raise ValueError( + f"task checkout commit mismatch: expected {config.expected_base_commit}, got {head}" + ) + if head_name != "HEAD": + raise ValueError("task checkout must have a detached HEAD") + if status: + raise ValueError("task checkout must be clean before the run") + + +def _untracked(worktree: Path) -> list[str]: + output = _git(worktree, "ls-files", "--others", "--exclude-standard", "-z") + paths = output.decode("utf-8", errors="surrogateescape").split("\0") + return sorted(path for path in paths if path and not _instrumentation_path(path)) + + +def _changed_paths(worktree: Path) -> list[str]: + tracked = _git(worktree, "diff", "--name-only", "-z", "HEAD", "--", ".") + decoded = tracked.decode("utf-8", errors="surrogateescape").split("\0") + return sorted( + { + path + for path in (*decoded, *_untracked(worktree)) + if path and not _instrumentation_path(path) + } + ) + + +def _auth_markers(auth_source: Path) -> tuple[bytes, ...]: + raw = auth_source.read_bytes() + markers: set[bytes] = {raw.strip()} if len(raw.strip()) >= 16 else set() + try: + decoded = json.loads(raw) + except json.JSONDecodeError: + decoded = None + + def collect(value: object) -> None: + if isinstance(value, str) and len(value.encode("utf-8")) >= 16: + markers.add(value.encode("utf-8")) + elif isinstance(value, dict): + for item in value.values(): + collect(item) + elif isinstance(value, list): + for item in value: + collect(item) + + collect(decoded) + return tuple(markers) + + +def _auth_material_exposed(worktree: Path, patch: bytes, auth_source: Path) -> bool: + markers = _auth_markers(auth_source) + if not markers: + return False + if any(marker in patch for marker in markers): + return True + for relative in _changed_paths(worktree): + candidate = (worktree / relative).resolve() + if not candidate.is_relative_to(worktree) or not candidate.is_file(): + continue + try: + content = candidate.read_bytes() + except OSError: + continue + if any(marker in content for marker in markers): + return True + return False + + +def _patch_and_stats(worktree: Path) -> tuple[bytes, int, int | None]: + patch_parts = [_git(worktree, "diff", "--binary", "HEAD", "--", ".", ":(exclude).codex")] + numstat_parts = [_git(worktree, "diff", "--numstat", "HEAD", "--", ".", ":(exclude).codex")] + for relative in _untracked(worktree): + patch_parts.append( + _git( + worktree, + "diff", + "--binary", + "--no-index", + "--", + "/dev/null", + relative, + allowed_codes=(0, 1), + ) + ) + numstat_parts.append( + _git( + worktree, + "diff", + "--numstat", + "--no-index", + "--", + "/dev/null", + relative, + allowed_codes=(0, 1), + ) + ) + patch = b"".join(patch_parts) + files: set[str] = set() + diff_lines = 0 + binary = False + for raw_line in b"".join(numstat_parts).splitlines(): + fields = raw_line.split(b"\t", maxsplit=2) + if len(fields) != 3: + continue + added, deleted, path = fields + files.add(path.decode("utf-8", errors="replace")) + if added == b"-" or deleted == b"-": + binary = True + else: + diff_lines += int(added) + int(deleted) + return patch, len(files), None if binary else diff_lines + + +def _empty_metrics() -> CodexMetrics: + return CodexMetrics(None, False, None, 0, 0, 0, 0, 0, 0, ()) + + +def _configuration_hash(config: RunConfig) -> str: + payload = { + "schema_version": 1, + "instance_id": config.instance_id, + "base_commit": config.expected_base_commit, + "condition": config.condition, + "repetition": config.repetition, + "model": config.model, + "reasoning_effort": config.reasoning_effort, + "codex_version": config.codex_version, + "sandbox": "workspace-write", + "network_access": False, + "timeout_seconds": config.timeout_seconds, + "prompt_sha256": hashlib.sha256(config.prompt.encode("utf-8")).hexdigest(), + "disabled_features": _FEATURES_DISABLED, + "marginal": ( + "absent" if config.condition == "baseline" else "balanced+diminishing-defaults+enforce" + ), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def run_task(config: RunConfig) -> dict[str, Any]: + """Run one already-materialized task and write local raw evidence plus one record.""" + + worktree = config.worktree.resolve() + run_dir = config.run_dir.resolve() + if not worktree.is_dir(): + raise ValueError(f"worktree does not exist: {worktree}") + if run_dir.is_relative_to(worktree): + raise ValueError("run directory must be outside the task checkout") + if not config.codex_executable.is_file(): + raise ValueError(f"Codex executable does not exist: {config.codex_executable}") + if not config.auth_source.is_file(): + raise ValueError(f"Codex auth source does not exist: {config.auth_source}") + if (worktree / ".codex" / "hooks.json").exists(): + raise ValueError("task checkout already contains .codex/hooks.json") + _validate_starting_checkout(config, worktree) + run_dir.mkdir(parents=True, exist_ok=False) + + events_path = run_dir / "codex-events.jsonl" + stderr_path = run_dir / "codex-stderr.log" + model_patch_path = run_dir / "model.patch" + hook_failure_path = run_dir / "hook-failures.log" + daemon_events_path = run_dir / "marginal-events.jsonl" + daemon_summary_path = run_dir / "marginal-summary.json" + daemon_stdout_path = run_dir / "marginal-daemon.stdout.log" + daemon_stderr_path = run_dir / "marginal-daemon.stderr.log" + + timed_out = False + exit_code: int | None = None + infrastructure_error: str | None = None + daemon_summary: dict[str, Any] | None = None + started = time.perf_counter_ns() + + with ( + tempfile.TemporaryDirectory(prefix="codex-home-") as codex_home_value, + tempfile.TemporaryDirectory(prefix="task-home-") as runtime_home_value, + tempfile.TemporaryDirectory(prefix="mg-", dir="/tmp") as socket_dir_value, + ): + codex_home = Path(codex_home_value) + auth_target = codex_home / "auth.json" + shutil.copyfile(config.auth_source, auth_target) + auth_target.chmod(0o600) + runtime_home = Path(runtime_home_value) + environment = _safe_environment(config, codex_home, runtime_home) + shell_environment: dict[str, str] = { + key: environment[key] + for key in ("HOME", "LANG", "LC_ALL", "LOGNAME", "PATH", "SHELL", "TMPDIR", "USER") + if key in environment + } + daemon: subprocess.Popen[bytes] | None = None + socket_path = Path(socket_dir_value) / "m.sock" + + try: + if config.condition == "marginal": + install_project_hooks( + worktree, + python_executable=Path(sys.executable), + hook_client=_HOOK_CLIENT, + ) + environment["MARGINAL_SOCKET"] = str(socket_path) + environment["MARGINAL_HOOK_FAILURE_LOG"] = str(hook_failure_path) + daemon_environment = dict(environment) + daemon_environment["PYTHONPATH"] = str(_SOURCE_ROOT) + with ( + daemon_stdout_path.open("wb") as daemon_stdout, + daemon_stderr_path.open("wb") as daemon_stderr, + ): + daemon = subprocess.Popen( + [ + sys.executable, + "-m", + "benchmark.codex_adapter.daemon", + "--socket", + str(socket_path), + "--events", + str(daemon_events_path), + "--summary", + str(daemon_summary_path), + ], + cwd=_SOURCE_ROOT, + env=daemon_environment, + stdout=daemon_stdout, + stderr=daemon_stderr, + ) + _wait_for_daemon(daemon, socket_path) + + with events_path.open("wb") as events, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + _command(config, shell_environment), + cwd=worktree, + env=environment, + stdin=subprocess.PIPE, + stdout=events, + stderr=stderr, + ) + try: + process.communicate( + config.prompt.encode("utf-8"), timeout=config.timeout_seconds + ) + except subprocess.TimeoutExpired: + timed_out = True + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + exit_code = process.returncode + else: + with events_path.open("wb") as events, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + _command(config, shell_environment), + cwd=worktree, + env=environment, + stdin=subprocess.PIPE, + stdout=events, + stderr=stderr, + ) + try: + process.communicate( + config.prompt.encode("utf-8"), timeout=config.timeout_seconds + ) + except subprocess.TimeoutExpired: + timed_out = True + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + exit_code = process.returncode + except Exception as exc: + infrastructure_error = f"{type(exc).__name__}: {exc}" + finally: + if daemon is not None: + if daemon.poll() is None: + daemon.terminate() + try: + daemon.wait(timeout=10) + except subprocess.TimeoutExpired: + daemon.kill() + daemon.wait(timeout=5) + if daemon_summary_path.is_file(): + try: + loaded = json.loads(daemon_summary_path.read_text(encoding="utf-8")) + if isinstance(loaded, dict): + daemon_summary = loaded + except json.JSONDecodeError: + infrastructure_error = infrastructure_error or "invalid daemon summary" + else: + infrastructure_error = infrastructure_error or "missing daemon summary" + + wall_time_ms = max(0, round((time.perf_counter_ns() - started) / 1_000_000)) + try: + metrics = parse_codex_jsonl(events_path) + except (OSError, EventParseError) as exc: + metrics = _empty_metrics() + infrastructure_error = infrastructure_error or f"event parse failed: {exc}" + + try: + patch, files_modified, diff_lines = _patch_and_stats(worktree) + except Exception as exc: + patch, files_modified, diff_lines = b"", 0, None + infrastructure_error = infrastructure_error or f"patch extraction failed: {exc}" + security_violation = _auth_material_exposed(worktree, patch, config.auth_source) + if security_violation: + patch = b"" + infrastructure_error = infrastructure_error or "authentication material exposure blocked" + model_patch_path.write_bytes(patch) + + run_status = "completed" + error_code: str | None = None + hook_coverage_missing = bool( + config.condition == "marginal" + and daemon_summary is not None + and int(daemon_summary.get("committed", 0)) != metrics.tool_calls + ) + if security_violation: + run_status = "security_failed" + error_code = "AUTH_MATERIAL_EXFILTRATED" + elif config.condition == "marginal" and ( + infrastructure_error is not None + or hook_failure_path.exists() + or daemon_summary is None + or daemon_summary.get("pending") != 0 + or hook_coverage_missing + ): + run_status = "integration_failed" + if hook_failure_path.exists(): + error_code = "HOOK_FAILURE" + elif hook_coverage_missing: + error_code = "HOOK_COVERAGE_MISSING" + elif daemon_summary is not None and daemon_summary.get("pending") != 0: + error_code = "ORPHANED_RESERVATION" + else: + error_code = "MARGINAL_INFRASTRUCTURE" + elif infrastructure_error is not None: + run_status = "codex_failed" + error_code = "RUNNER_INFRASTRUCTURE" + elif timed_out: + run_status = "timeout" + error_code = "CODEX_TIMEOUT" + elif exit_code != 0: + run_status = "codex_failed" + error_code = f"CODEX_EXIT_{exit_code}" + elif not metrics.completed or metrics.tokens is None: + run_status = "codex_failed" + error_code = "TOKEN_TELEMETRY_MISSING" + elif metrics.errors: + run_status = "codex_failed" + error_code = "CODEX_STREAM_ERROR" + + if daemon_summary is None: + interventions = { + "recommended_denies": 0, + "applied_denies": 0, + "reviewed": 0, + "false_stops": 0, + } + governance = {"tokens": 0, "usd": 0.0, "latency_ms": 0.0} + else: + raw_interventions = daemon_summary.get("interventions", {}) + raw_governance = daemon_summary.get("governance", {}) + interventions = { + "recommended_denies": int(raw_interventions.get("recommended_denies", 0)), + "applied_denies": int(raw_interventions.get("applied_denies", 0)), + "reviewed": int(raw_interventions.get("reviewed", 0)), + "false_stops": int(raw_interventions.get("false_stops", 0)), + } + governance = { + "tokens": int(raw_governance.get("external_tokens", 0)), + "usd": float(raw_governance.get("external_usd", 0.0)), + "latency_ms": float(raw_governance.get("total_latency_ms", 0.0)), + } + + tokens: dict[str, int | None] + if metrics.tokens is None: + tokens = { + "input": None, + "cached_input": None, + "output": None, + "reasoning": None, + "total": None, + } + else: + tokens = {key: value for key, value in metrics.tokens.items()} + record: dict[str, Any] = { + "schema_version": 1, + "instance_id": config.instance_id, + "condition": config.condition, + "repetition": config.repetition, + "run_status": run_status, + "resolved": None, + "configuration_sha256": _configuration_hash(config), + "patch_sha256": hashlib.sha256(patch).hexdigest(), + "tokens": tokens, + "wall_time_ms": wall_time_ms, + "tool_calls": metrics.tool_calls, + "shell_commands": metrics.shell_commands, + "file_operations": metrics.file_operations, + "searches": metrics.searches, + "test_executions": metrics.test_executions, + "repeated_calls": metrics.repeated_calls, + "files_modified": files_modified, + "diff_lines": diff_lines, + "model_reroutes": None, + "interventions": interventions, + "governance": governance, + "error_code": error_code, + } + if infrastructure_error is not None: + (run_dir / "runner-error.txt").write_text(infrastructure_error + "\n", encoding="utf-8") + (run_dir / "run-record.json").write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return record diff --git a/benchmark/codex_adapter/workspace.py b/benchmark/codex_adapter/workspace.py new file mode 100644 index 0000000..0bba347 --- /dev/null +++ b/benchmark/codex_adapter/workspace.py @@ -0,0 +1,86 @@ +"""Deterministically fingerprint the task workspace observed by Codex hooks.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from pathlib import Path, PurePosixPath +from typing import Protocol + +_IGNORED_PARTS = frozenset( + { + ".codex", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + } +) + + +def _git(repo: Path, *args: str) -> bytes: + completed = subprocess.run( + ["git", *args], + cwd=repo, + check=False, + capture_output=True, + timeout=30, + env={**os.environ, "LC_ALL": "C"}, + ) + if completed.returncode != 0: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise ValueError(f"workspace is not a usable Git repository: {detail}") + return completed.stdout + + +def _root(path: Path) -> Path: + if not path.is_dir(): + raise ValueError(f"workspace path is not a directory: {path}") + output = _git(path, "rev-parse", "--show-toplevel") + return Path(output.decode("utf-8").strip()).resolve() + + +def _ignored(relative_path: str) -> bool: + return bool(_IGNORED_PARTS.intersection(PurePosixPath(relative_path).parts)) + + +def _update_chunk(digest: AnyHash, label: bytes, value: bytes) -> None: + digest.update(len(label).to_bytes(8, "big")) + digest.update(label) + digest.update(len(value).to_bytes(8, "big")) + digest.update(value) + + +class AnyHash(Protocol): + """Small structural type substitute compatible with hashlib objects on Python 3.10.""" + + def update(self, value: bytes) -> None: ... + + +def workspace_state_hash(path: str | Path) -> str: + """Hash HEAD, tracked changes, and material untracked files for one Git worktree.""" + + repo = _root(Path(path).resolve()) + head = _git(repo, "rev-parse", "HEAD").strip() + tracked_diff = _git(repo, "diff", "--binary", "HEAD", "--") + untracked = _git(repo, "ls-files", "--others", "--exclude-standard", "-z") + + digest = hashlib.sha256() + _update_chunk(digest, b"head", head) + _update_chunk(digest, b"tracked-diff", tracked_diff) + for encoded_path in sorted(filter(None, untracked.split(b"\0"))): + relative = encoded_path.decode("utf-8", errors="surrogateescape") + if _ignored(relative): + continue + file_path = repo / relative + if file_path.is_symlink(): + contents = os.readlink(file_path).encode("utf-8", errors="surrogateescape") + elif file_path.is_file(): + contents = file_path.read_bytes() + else: + continue + _update_chunk(digest, b"untracked-path", encoded_path) + _update_chunk(digest, b"untracked-content", contents) + return digest.hexdigest() diff --git a/benchmark/container/Dockerfile.tools b/benchmark/container/Dockerfile.tools new file mode 100644 index 0000000..227e509 --- /dev/null +++ b/benchmark/container/Dockerfile.tools @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 +ARG TASK_IMAGE +FROM --platform=linux/amd64 node:22-bookworm-slim AS codex_builder + +ARG CODEX_VERSION=0.147.0 +RUN npm install --global --omit=dev "@openai/codex@${CODEX_VERSION}" \ + && codex --version + +FROM ${TASK_IMAGE} + +ARG TASK_IMAGE +ARG CODEX_VERSION=0.147.0 +ARG MARGINAL_SOURCE_COMMIT + +LABEL org.opencontainers.image.title="MARGINAL Codex SWE-bench tool layer" \ + org.opencontainers.image.source="https://github.com/SignalLayerLabs/Marginal" \ + org.marginal.codex.version="${CODEX_VERSION}" \ + org.marginal.source.commit="${MARGINAL_SOURCE_COMMIT}" \ + org.marginal.task.image="${TASK_IMAGE}" + +COPY --from=codex_builder /usr/local/bin/node /opt/marginal-tools/bin/node +COPY --from=codex_builder /usr/local/lib/node_modules/@openai/codex /opt/marginal-tools/lib/codex +COPY benchmark/container/codex-wrapper.sh /opt/marginal-tools/bin/codex + +COPY pyproject.toml /opt/marginal/pyproject.toml +COPY src/marginal /opt/marginal/src/marginal +COPY benchmark/__init__.py /opt/marginal/benchmark/__init__.py +COPY benchmark/codex_adapter /opt/marginal/benchmark/codex_adapter +COPY benchmark/container/entrypoint.sh /opt/marginal/benchmark/container/entrypoint.sh + +RUN chmod 0555 \ + /opt/marginal-tools/bin/codex \ + /opt/marginal/benchmark/container/entrypoint.sh \ + && PYTHONPATH=/opt/marginal/src:/opt/marginal /usr/bin/python3 -c \ + "from marginal import Treasury; from benchmark.codex_adapter.engine import CodexGovernanceEngine" \ + && /opt/marginal-tools/bin/codex --version + +ENV PYTHONPATH=/opt/marginal/src:/opt/marginal +WORKDIR /testbed diff --git a/benchmark/container/codex-wrapper.sh b/benchmark/container/codex-wrapper.sh new file mode 100644 index 0000000..0e964ef --- /dev/null +++ b/benchmark/container/codex-wrapper.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +exec /opt/marginal-tools/bin/node /opt/marginal-tools/lib/codex/bin/codex.js "$@" + diff --git a/benchmark/container/entrypoint.sh b/benchmark/container/entrypoint.sh new file mode 100644 index 0000000..21abe71 --- /dev/null +++ b/benchmark/container/entrypoint.sh @@ -0,0 +1,158 @@ +#!/bin/bash +set -Eeuo pipefail + +umask 077 + +required=( + MARGINAL_INSTANCE_ID + MARGINAL_CONDITION + MARGINAL_EXPECTED_BASE_COMMIT + MARGINAL_MODEL + MARGINAL_REASONING_EFFORT + MARGINAL_TIMEOUT_SECONDS +) +for name in "${required[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "missing required environment: ${name}" >&2 + exit 64 + fi +done +if [[ "${MARGINAL_CONDITION}" != "baseline" && "${MARGINAL_CONDITION}" != "marginal" ]]; then + echo "invalid MARGINAL_CONDITION" >&2 + exit 64 +fi + +mkdir -p /marginal-home/.codex /marginal-home/home +install -m 0600 /run/secrets/codex-auth.json /marginal-home/.codex/auth.json +export CODEX_HOME=/marginal-home/.codex +export HOME=/marginal-home/home +export PYTHONPATH=/opt/marginal/src:/opt/marginal +export MARGINAL_SOCKET=/marginal-home/marginal.sock +export MARGINAL_HOOK_FAILURE_LOG=/marginal-run/hook-failures.log + +if ! git -C /testbed cat-file -e "${MARGINAL_EXPECTED_BASE_COMMIT}^{commit}"; then + echo "expected base commit is absent from task image" >&2 + exit 65 +fi +git -C /testbed checkout --detach --force "${MARGINAL_EXPECTED_BASE_COMMIT}" +git -C /testbed reset --hard "${MARGINAL_EXPECTED_BASE_COMMIT}" +git -C /testbed clean -ffd +if [[ "$(git -C /testbed rev-parse HEAD)" != "${MARGINAL_EXPECTED_BASE_COMMIT}" ]]; then + echo "task checkout did not reach expected base commit" >&2 + exit 65 +fi +if [[ -n "$(git -C /testbed status --porcelain --untracked-files=all)" ]]; then + echo "task checkout is not clean" >&2 + exit 65 +fi + +source /opt/miniconda3/etc/profile.d/conda.sh +conda activate testbed + +daemon_pid="" +cleanup() { + if [[ -n "${daemon_pid}" ]] && kill -0 "${daemon_pid}" 2>/dev/null; then + kill -TERM "${daemon_pid}" 2>/dev/null || true + wait "${daemon_pid}" 2>/dev/null || true + fi +} +trap cleanup EXIT TERM INT + +if [[ "${MARGINAL_CONDITION}" == "marginal" ]]; then + /usr/bin/python3 -m benchmark.codex_adapter.daemon \ + --socket "${MARGINAL_SOCKET}" \ + --events /marginal-run/treasury-events.jsonl \ + --summary /marginal-run/daemon-summary.json \ + > /marginal-run/daemon.stdout.log \ + 2> /marginal-run/daemon.stderr.log & + daemon_pid=$! + + ready=0 + for _ in $(seq 1 200); do + if [[ -S "${MARGINAL_SOCKET}" ]]; then + ready=1 + break + fi + if ! kill -0 "${daemon_pid}" 2>/dev/null; then + break + fi + sleep 0.05 + done + if [[ "${ready}" != "1" ]]; then + echo "MARGINAL daemon did not become ready" >&2 + exit 70 + fi + + /usr/bin/python3 -c \ + 'from benchmark.codex_adapter.hook_config import install_project_hooks; install_project_hooks("/marginal-home", python_executable="/usr/bin/python3", hook_client="/opt/marginal/benchmark/codex_adapter/hook_client.py")' +fi + +shell_policy="$(/usr/bin/python3 -c 'import json, os; values={"HOME":"/marginal-home/home","LANG":"C.UTF-8","LC_ALL":"C.UTF-8","PATH":os.environ["PATH"],"TERM":"dumb"}; print("{" + ",".join(f"{key}={json.dumps(value)}" for key, value in sorted(values.items())) + "}")')" + +command=( + /opt/marginal-tools/bin/codex exec + --ignore-rules + --ephemeral + --json + --color never + --strict-config + --dangerously-bypass-hook-trust + --enable codex_hooks + -m "${MARGINAL_MODEL}" + -c "model_reasoning_effort=\"${MARGINAL_REASONING_EFFORT}\"" + -c 'approval_policy="never"' + -c 'sandbox_workspace_write.network_access=false' + -c 'shell_environment_policy.inherit="none"' + -c 'shell_environment_policy.ignore_default_excludes=false' + -c "shell_environment_policy.set=${shell_policy}" + -s workspace-write + -C /testbed +) +disabled_features=( + apps + browser_use + browser_use_external + browser_use_full_cdp_access + computer_use + goals + image_generation + in_app_browser + multi_agent + multi_agent_v2 + plugins + search_tool + skill_search + standalone_web_search + tool_suggest + web_search_request + workspace_dependencies +) +for feature in "${disabled_features[@]}"; do + command+=(--disable "${feature}") +done +command+=(-) + +set +e +timeout --signal=TERM --kill-after=30s "${MARGINAL_TIMEOUT_SECONDS}s" \ + "${command[@]}" \ + < /marginal-run/prompt.txt \ + > /marginal-run/codex-events.jsonl \ + 2> /marginal-run/codex-stderr.log +codex_status=$? +set -e + +git -C /testbed diff --binary HEAD -- . ':(exclude).codex' > /marginal-run/model.patch +git -C /testbed diff --numstat HEAD -- . ':(exclude).codex' > /marginal-run/model.numstat +git -C /testbed status --porcelain --untracked-files=all -- . ':(exclude).codex' \ + > /marginal-run/worktree.status +printf '{"codex_exit_code":%d,"condition":"%s","instance_id":"%s"}\n' \ + "${codex_status}" "${MARGINAL_CONDITION}" "${MARGINAL_INSTANCE_ID}" \ + > /marginal-run/container-status.json + +if [[ -n "${daemon_pid}" ]]; then + kill -TERM "${daemon_pid}" 2>/dev/null || true + wait "${daemon_pid}" || true + daemon_pid="" +fi + +exit "${codex_status}" diff --git a/benchmark/environment.json b/benchmark/environment.json new file mode 100644 index 0000000..4dcf5c4 --- /dev/null +++ b/benchmark/environment.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-11", + "timezone": "Europe/Rome", + "repository": { + "url": "https://github.com/SignalLayerLabs/Marginal.git", + "commit": "4c8856401b4c752d5c214df5e84b9632d9897ec9", + "version": "0.2.0+unreleased" + }, + "host": { + "operating_system": "macOS 26.6", + "build": "25G72", + "architecture": "arm64" + }, + "python": "3.13.0", + "codex": { + "cli_version": "0.147.0", + "model": "gpt-5.6-sol", + "reasoning_effort": "high", + "sandbox": "workspace-write", + "timeout_seconds": 1800, + "json_events": true, + "ephemeral": true + }, + "benchmark": { + "name": "swe-bench-lite-dev-smoke", + "dataset": "princeton-nlp/SWE-bench_Lite", + "split": "dev", + "task_set_sha256": "1c8c06935484fd57f4c27c363418884542bcfb06c69d02f480b61fe7687b58eb", + "prompt_template_sha256": "3a2ced1afb7e9a74176090db264259c52b6ebcf383d787a1030f83ecb6d67416", + "verifier": "official SWE-bench harness; backend selected by preflight", + "verifier_version": null + }, + "credentials_recorded": false +} diff --git a/benchmark/marginal_system_description.md b/benchmark/marginal_system_description.md new file mode 100644 index 0000000..bf6b27d --- /dev/null +++ b/benchmark/marginal_system_description.md @@ -0,0 +1,123 @@ +# What MARGINAL Actually Does + +## Scope inspected + +This description is based on repository commit +`4c8856401b4c752d5c214df5e84b9632d9897ec9`, including the implementation under +`src/marginal`, tests, Universal Agent Protocol, governance controls, and benchmark code. + +MARGINAL is a provider-neutral Python decision, accounting, and evidence layer for agent +runtimes. It is not itself a coding agent, prompt compressor, model router, or completed +Codex integration. + +## Implemented pipeline + +```text +engine adapter proposes normalized action + -> Treasury prepares identity and checks pending work + -> hard child/parent budget checks + -> MarginalPolicy requests ValueEstimator estimate + -> optional DiminishingReturnDetector adjustment/rejection + -> expected value, cost value, score, and ROI decision + -> execution mode applies or overrides recommendation + -> reserve + -> external action executes or is skipped + -> commit actual usage, settle measured failure, or abort + -> trace/Decision Ledger/outcome/governance evidence +``` + +## Inputs and signals + +An `Action` contains a name, kind, estimated `Cost`, optional expected success gain, +current success probability, verification flag, fingerprint, and metadata. `Cost` has +tokens, direct USD, latency, and application-defined risk. The core does not inspect a +Codex transcript or infer these fields from native Codex events; an adapter must supply +them. + +The reference `ValueEstimator` uses, in priority order: + +1. explicit `Action.expected_gain`; +2. contextual historical observations; +3. action-kind historical observations; +4. a default expected gain of `0.05` with zero confidence. + +Action-level realized gain is accepted only through an explicit observation call. Task +success is not automatically assigned as causal credit to every action in a trajectory. + +## Policy + +The policy computes: + +```text +expected value = capped expected gain * configured outcome value +cost value = direct USD + token shadow cost + latency shadow cost + risk shadow cost +score = expected value - cost value +ROI = expected value / cost value +``` + +It can reject work because the target success probability is reached, the action is an +executed or pending duplicate, a budget/reserve would be breached, expected gain is too +low, or score/ROI does not clear the configured threshold. `fund_best` ranks a caller- +provided candidate set; it does not invent alternatives for a single action proposed by +an external agent. + +## Diminishing returns + +The opt-in detector derives a semantic key from +`metadata.marginal_semantic_key`, or from normalized kind/name/phase. It also reads +`state_hash` and optional `evidence_hash` metadata. + +It fails open when state is absent. At the default `gain_decay=0.5` and +`max_same_state_repeats=2`, one successful same-state execution causes the next proposal's +gain to be halved; after two such executions, the next proposal receives a stop +recommendation. A changed state or changed non-empty evidence hash resets the repetition +pressure. Only successful execution advances detector history. + +This is exact adapter-defined semantic repetition, not fuzzy recognition of arbitrary +reasoning loops or hypothesis oscillation. + +## State and evidence + +`Treasury` maintains committed usage, reservations, pending semantic identities, executed +fingerprints, counts, outcomes, estimator observations, and a shared governance tracker. +The Decision Ledger records run/task/trajectory/engine/model identity, recommendation +versus applied behavior, policy and estimator identity, cost, failures, and outcomes. + +Governance evidence separates local policy-decision latency from external adapter tokens, +USD, and latency. False stops require explicit external review; they are not inferred from +final task outcomes. + +## Modes + +- `shadow`: recommendations are recorded but proposed actions execute. +- `recommend`: currently non-blocking like shadow at the Treasury authorization boundary. +- `enforce`: denials prevent guarded work from executing. + +`fund_best` remains an active selection API in every mode. + +## Codex integration status + +The repository supplies Universal Agent Protocol and a generic runtime, but no completed +Codex adapter. The existing SWE-bench workflow verifies already-generated patches and +merges telemetry; it does not run Codex or inject MARGINAL into its loop. + +For this experiment, the missing integration is implemented outside the core through +official Codex `PreToolUse` and `PostToolUse` hooks. Therefore the tested variable is the +current MARGINAL policy applied to hook-covered local tool actions. Hosted tools and model +calls are outside the initial enforcement boundary. + +The pinned Codex 0.147 stable response for unified shell execution omits process exit +status from `PostToolUse`. The adapter therefore cannot faithfully identify successful +shell/test/search executions. It failure-settles them conservatively and the mandatory +preflight blocks comparative inference until a tested success-observable boundary exists. + +## OFF and ON conditions + +**OFF:** Codex runs with the external JSONL collector but without MARGINAL imports, +processes, hooks, state, messages, or configuration. + +**ON:** the identical Codex invocation additionally loads the tested hook adapter and a +per-task MARGINAL daemon configured with the released balanced profile, default +diminishing-return detector, unlimited hard budgets, and enforce mode. + +No core behavior or threshold may be changed after comparative outcomes are observed. diff --git a/benchmark/methodology.md b/benchmark/methodology.md new file mode 100644 index 0000000..5720611 --- /dev/null +++ b/benchmark/methodology.md @@ -0,0 +1,149 @@ +# Frozen Methodology + +Status: preregistered before any Codex OFF/ON benchmark execution. + +Feasibility amendment (2026-08-11, before inference): the pinned Codex 0.147 hook payload +does not expose shell exit status. Shell and verification actions therefore cannot be +counted as successful without guessing and are conservatively failure-settled. Because +those actions are the intended repeat-control surface, comparative execution remains +blocked pending a success-observable interception boundary. The official per-instance +SWE-bench execution environment is also mandatory, not only the final verifier. + +## Hypotheses + +Primary: enabling the frozen MARGINAL integration does not reduce independently verified +task correctness and may change paired resolution outcomes. + +Secondary: when correctness is preserved, MARGINAL may reduce effective tokens, actions, +wall time, and repeated same-state work after including governance overhead. + +A negative, mixed, pass-through, or underpowered result is valid. + +## Conditions + +| Dimension | Baseline | MARGINAL | +|---|---|---| +| Codex CLI | `0.147.0` | `0.147.0` | +| Model | `gpt-5.6-sol` | `gpt-5.6-sol` | +| Reasoning effort | `high` | `high` | +| Task checkout | task `base_commit` | same task `base_commit` | +| Prompt template | frozen SHA-256 | identical | +| Sandbox | `workspace-write` | `workspace-write` | +| Timeout | 1,800 seconds | 1,800 seconds | +| Hosted tools | disabled | disabled | +| JSONL observer | enabled | enabled | +| MARGINAL | absent | hook adapter + daemon | + +The ON policy is `balanced` plus `DiminishingReturnDetector()` defaults, unlimited hard +budgets, and `enforce` mode. Tool cost is zero at authorization because no defensible +future-token forecast is available at the hook boundary. The measured experiment therefore +tests repetition control rather than a calibrated economic-ROI claim. + +## Isolation + +Each task, condition, and repetition starts in a separately materialized checkout at the +task's base commit. Runtime directories, Codex session state, MARGINAL ledger, generated +patches, and test caches are unique. No lane may read another lane's trajectory, patch, +notes, or result. + +Codex tool execution must occur in the official pinned per-instance SWE-bench environment. +A bare host checkout is insufficient because missing or divergent dependencies can change +the debugging trajectory and MARGINAL interventions. + +Task order is frozen. Where concurrency is used, condition is alternated within task and +the scheduling record is retained. An ON checkout is never created from an OFF checkout. + +## Smoke task selection + +Dataset: `princeton-nlp/SWE-bench_Lite`, split `dev`, frozen 2026-08-07. + +The smoke set is the three lexicographically lowest SHA-256 digests of `instance_id`: + +1. `pvlib__pvlib-python-1072` +2. `pvlib__pvlib-python-1707` +3. `pylint-dev__astroid-1978` + +Selection occurred before any OFF/ON outcome. Smoke results validate the pipeline and are +not headline performance evidence. + +## Primary outcome + +The official SWE-bench verifier supplies `resolved`. A task pair is classified as: + +- `BOTH_SOLVE` +- `BASELINE_ONLY` +- `MARGINAL_ONLY` +- `NEITHER` + +Resolution counts and rate appear before all efficiency metrics. Empty patches, timeouts, +Codex failures, premature stops, integration failures, and evaluation errors remain +unresolved unless the official verifier reports resolution. + +## Secondary metrics + +Measured when available: + +- input, cached input, output, reasoning, and total tokens; +- agent tokens and effective tokens including governance; +- wall-clock and tool latency; +- direct and governance USD when actually observable; +- tool calls, shell commands, file operations, searches, and test executions; +- repeated equivalent actions; +- files modified and diff line counts; +- MARGINAL decisions, deny recommendations, applied denies, and decision index; +- hook/adapter failures and model reroutes. + +Unavailable fields are `null`, never estimated as zero unless zero is directly known. +For Codex CLI telemetry, `total tokens` is `input_tokens + output_tokens`; cached input is +reported as a subset of input and reasoning output as a subset of output, so neither is added +again. This accounting rule is frozen before execution. + +## Frozen repeat definition + +A proposed action is an unproductive-repeat candidate when all conditions hold: + +1. its canonical tool name and normalized input produce the same semantic key; +2. the previous equivalent action completed successfully; +3. the observable workspace state hash is unchanged; +4. no new evidence hash exists; +5. executing it produces no verifier-relevant progress. + +Conditions 1-4 are machine-detected. Condition 5 is retained for retrospective review and +is not used to change detector thresholds. + +Under Codex 0.147, a shell `PostToolUse` response lacks the exit status needed for condition +2. Shell and verification calls are therefore failure-settled and excluded from successful +repeat history; this limitation is a full-preflight blocker, not a benchmark result. + +## Intervention review + +Every applied or would-apply deny is reviewed after execution using frozen labels: + +- `BENEFICIAL`: explicit evidence shows the skipped action was redundant and the required + outcome was preserved or reached without replacement cost exceeding the saving; +- `NEUTRAL`: no material trajectory, correctness, or effective-cost change is supported; +- `HARMFUL`: explicit evidence shows the action would have contributed to correctness or + the denial caused greater replacement cost; +- `INDETERMINATE`: counterfactual evidence is insufficient. + +Task success alone cannot label an intervention beneficial. Task failure alone cannot +label it harmful. Reviewers should not see aggregate condition results while labeling. + +## Analysis + +The three-task smoke and 20-task canary are exploratory. They do not support statistical +significance or a one-percentage-point non-inferiority claim. + +For the later repeated experiment, report task-clustered paired bootstrap intervals for +resolution delta and continuous metrics. Report an exact paired binary analysis for the +predeclared task-level correctness summary. Continuous metrics include mean, median, p25, +p75, minimum, and maximum. Efficiency is shown overall and separately for `BOTH_SOLVE`. + +## Stop rules + +Stop before larger runs if any smoke task has missing token telemetry, an orphaned +reservation, a hook failure, unmatched configuration, cross-lane contamination, a missing +prediction row, or a verifier pipeline failure. Do not remove the failing task. + +After the smoke, project the 20-task canary cost from measured effective tokens and elapsed +time. No larger run begins without a complete smoke report. diff --git a/benchmark/prompt_template.txt b/benchmark/prompt_template.txt new file mode 100644 index 0000000..0509a2d --- /dev/null +++ b/benchmark/prompt_template.txt @@ -0,0 +1,12 @@ +You are fixing one software issue in the repository provided as your working directory. + +Read the issue below, inspect the implementation and tests, make the smallest complete fix, +and run relevant tests. Do not access benchmark metadata, reference patches, gold patches, +or another run's artifacts. Do not commit or push. + +ISSUE: +{{problem_statement}} + +When finished, leave the working tree containing only the proposed solution and any tests +that are legitimately part of that solution. + diff --git a/benchmark/results/baseline.jsonl b/benchmark/results/baseline.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/results/marginal.jsonl b/benchmark/results/marginal.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/results/paired_results.csv b/benchmark/results/paired_results.csv new file mode 100644 index 0000000..c858c22 --- /dev/null +++ b/benchmark/results/paired_results.csv @@ -0,0 +1,2 @@ +instance_id,repetition,baseline_resolved,marginal_resolved,paired_outcome,baseline_tokens,marginal_effective_tokens,delta_effective_tokens + diff --git a/benchmark/schemas/run-record-v1.json b/benchmark/schemas/run-record-v1.json new file mode 100644 index 0000000..7604a3f --- /dev/null +++ b/benchmark/schemas/run-record-v1.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://signallayerlabs.github.io/Marginal/schemas/codex-run-record-v1.json", + "title": "Codex MARGINAL benchmark run record", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "instance_id", + "condition", + "repetition", + "run_status", + "resolved", + "configuration_sha256", + "patch_sha256", + "tokens", + "wall_time_ms", + "tool_calls", + "repeated_calls", + "interventions", + "governance" + ], + "properties": { + "schema_version": {"const": 1}, + "instance_id": {"type": "string", "minLength": 1}, + "condition": {"enum": ["baseline", "marginal"]}, + "repetition": {"type": "integer", "minimum": 1}, + "run_status": { + "enum": [ + "completed", + "timeout", + "codex_failed", + "integration_failed", + "security_failed", + "materialization_failed", + "verifier_failed" + ] + }, + "resolved": {"type": ["boolean", "null"]}, + "configuration_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "patch_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "tokens": { + "type": "object", + "additionalProperties": false, + "required": ["input", "cached_input", "output", "reasoning", "total"], + "properties": { + "input": {"type": ["integer", "null"], "minimum": 0}, + "cached_input": {"type": ["integer", "null"], "minimum": 0}, + "output": {"type": ["integer", "null"], "minimum": 0}, + "reasoning": {"type": ["integer", "null"], "minimum": 0}, + "total": {"type": ["integer", "null"], "minimum": 0} + } + }, + "wall_time_ms": {"type": "integer", "minimum": 0}, + "tool_calls": {"type": "integer", "minimum": 0}, + "shell_commands": {"type": ["integer", "null"], "minimum": 0}, + "file_operations": {"type": ["integer", "null"], "minimum": 0}, + "searches": {"type": ["integer", "null"], "minimum": 0}, + "test_executions": {"type": ["integer", "null"], "minimum": 0}, + "repeated_calls": {"type": "integer", "minimum": 0}, + "files_modified": {"type": ["integer", "null"], "minimum": 0}, + "diff_lines": {"type": ["integer", "null"], "minimum": 0}, + "model_reroutes": {"type": ["integer", "null"], "minimum": 0}, + "interventions": { + "type": "object", + "additionalProperties": false, + "required": ["recommended_denies", "applied_denies", "reviewed", "false_stops"], + "properties": { + "recommended_denies": {"type": "integer", "minimum": 0}, + "applied_denies": {"type": "integer", "minimum": 0}, + "reviewed": {"type": "integer", "minimum": 0}, + "false_stops": {"type": "integer", "minimum": 0} + } + }, + "governance": { + "type": "object", + "additionalProperties": false, + "required": ["tokens", "usd", "latency_ms"], + "properties": { + "tokens": {"type": ["integer", "null"], "minimum": 0}, + "usd": {"type": ["number", "null"], "minimum": 0}, + "latency_ms": {"type": ["number", "null"], "minimum": 0} + } + }, + "error_code": {"type": ["string", "null"]} + } +} diff --git a/benchmark/scripts/__init__.py b/benchmark/scripts/__init__.py new file mode 100644 index 0000000..0146292 --- /dev/null +++ b/benchmark/scripts/__init__.py @@ -0,0 +1 @@ +"""Command-line entrypoints for the Codex A/B benchmark.""" diff --git a/benchmark/scripts/preflight.py b/benchmark/scripts/preflight.py new file mode 100644 index 0000000..4dd580a --- /dev/null +++ b/benchmark/scripts/preflight.py @@ -0,0 +1,54 @@ +"""Run the frozen Codex/SWE-bench readiness gate.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from benchmark.codex_adapter.preflight import PreflightConfig, PreflightError, run_preflight + +_ROOT = Path(__file__).resolve().parents[2] + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--codex", type=Path, default=Path("/opt/homebrew/bin/codex")) + parser.add_argument("--python", type=Path, default=_ROOT / ".venv" / "bin" / "python") + parser.add_argument("--auth", type=Path, default=Path.home() / ".codex" / "auth.json") + parser.add_argument("--output", type=Path) + parser.add_argument("--skip-verifier", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + environment = json.loads((_ROOT / "benchmark" / "environment.json").read_text()) + try: + report = run_preflight( + PreflightConfig( + repository_root=_ROOT, + codex_executable=args.codex, + python_executable=args.python, + auth_source=args.auth, + environment_path=_ROOT / "benchmark" / "environment.json", + tasks_path=_ROOT / "benchmark" / "tasks.json", + prompt_path=_ROOT / "benchmark" / "prompt_template.txt", + schema_path=_ROOT / "benchmark" / "schemas" / "run-record-v1.json", + expected_repository_commit=environment["repository"]["commit"], + require_verifier=not args.skip_verifier, + require_task_environment=not args.skip_verifier, + ) + ) + except PreflightError as exc: + raise SystemExit(f"preflight failed: {exc}") from None + encoded = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(encoded, encoding="utf-8") + print(encoded, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/prepare_swebench_tasks.py b/benchmark/scripts/prepare_swebench_tasks.py new file mode 100644 index 0000000..7348a76 --- /dev/null +++ b/benchmark/scripts/prepare_swebench_tasks.py @@ -0,0 +1,36 @@ +"""Fetch and materialize the frozen SWE-bench Lite smoke tasks.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from benchmark.codex_adapter.dataset import fetch_frozen_tasks, materialize_task + +_ROOT = Path(__file__).resolve().parents[2] + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", required=True, type=Path) + parser.add_argument("--instance-id", action="append") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + tasks = fetch_frozen_tasks(_ROOT / "benchmark" / "tasks.json") + selected = set(args.instance_id or (task.instance_id for task in tasks)) + unknown = selected - {task.instance_id for task in tasks} + if unknown: + raise SystemExit(f"unknown frozen task IDs: {sorted(unknown)}") + for task in tasks: + if task.instance_id in selected: + target = args.destination / task.instance_id + materialize_task(task, target) + print(f"{task.instance_id}\t{task.base_commit}\t{target}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/run_codex_task.py b/benchmark/scripts/run_codex_task.py new file mode 100644 index 0000000..4f16b0f --- /dev/null +++ b/benchmark/scripts/run_codex_task.py @@ -0,0 +1,73 @@ +"""Run one frozen SWE-bench task in one Codex benchmark condition.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from benchmark.codex_adapter.dataset import fetch_frozen_tasks, render_prompt +from benchmark.codex_adapter.preflight import PreflightConfig, PreflightError, run_preflight +from benchmark.codex_adapter.runner import RunConfig, run_task + +_ROOT = Path(__file__).resolve().parents[2] + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--instance-id", required=True) + parser.add_argument("--condition", required=True, choices=("baseline", "marginal")) + parser.add_argument("--worktree", required=True, type=Path) + parser.add_argument("--run-dir", required=True, type=Path) + parser.add_argument("--codex", type=Path, default=Path("/opt/homebrew/bin/codex")) + parser.add_argument("--auth", type=Path, default=Path.home() / ".codex" / "auth.json") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + environment_path = _ROOT / "benchmark" / "environment.json" + environment = json.loads(environment_path.read_text(encoding="utf-8")) + try: + run_preflight( + PreflightConfig( + repository_root=_ROOT, + codex_executable=args.codex, + python_executable=Path(sys.executable), + auth_source=args.auth, + environment_path=environment_path, + tasks_path=_ROOT / "benchmark" / "tasks.json", + prompt_path=_ROOT / "benchmark" / "prompt_template.txt", + schema_path=_ROOT / "benchmark" / "schemas" / "run-record-v1.json", + expected_repository_commit=environment["repository"]["commit"], + require_verifier=True, + require_task_environment=True, + ) + ) + except PreflightError as exc: + raise SystemExit(f"benchmark run blocked: {exc}") from None + tasks = fetch_frozen_tasks(_ROOT / "benchmark" / "tasks.json") + task = next((item for item in tasks if item.instance_id == args.instance_id), None) + if task is None: + raise SystemExit(f"unknown frozen task: {args.instance_id}") + prompt = render_prompt(_ROOT / "benchmark" / "prompt_template.txt", task.problem_statement) + record = run_task( + RunConfig( + instance_id=task.instance_id, + condition=args.condition, + repetition=1, + worktree=args.worktree, + expected_base_commit=task.base_commit, + run_dir=args.run_dir, + prompt=prompt, + codex_executable=args.codex, + auth_source=args.auth, + ) + ) + print(json.dumps(record, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/run_container_task.py b/benchmark/scripts/run_container_task.py new file mode 100644 index 0000000..f2679ba --- /dev/null +++ b/benchmark/scripts/run_container_task.py @@ -0,0 +1,63 @@ +"""Run one frozen Codex benchmark lane in its pinned SWE-bench container.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from benchmark.codex_adapter.container_runner import ContainerTaskConfig, run_container_task +from benchmark.codex_adapter.dataset import fetch_frozen_tasks, render_prompt + +_ROOT = Path(__file__).resolve().parents[2] + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--instance-id", required=True) + parser.add_argument("--condition", choices=("baseline", "marginal"), required=True) + parser.add_argument("--task-image", required=True) + parser.add_argument("--overlay-image", required=True) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--repetition", type=int, default=1) + parser.add_argument("--timeout-seconds", type=int, default=1800) + parser.add_argument("--auth", type=Path, default=Path.home() / ".codex" / "auth.json") + parser.add_argument("--tasks", type=Path, default=_ROOT / "benchmark" / "tasks.json") + parser.add_argument( + "--prompt-template", + type=Path, + default=_ROOT / "benchmark" / "prompt_template.txt", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + tasks = fetch_frozen_tasks(args.tasks) + task = next((item for item in tasks if item.instance_id == args.instance_id), None) + if task is None: + raise SystemExit(f"unknown frozen task: {args.instance_id}") + prompt = render_prompt(args.prompt_template, task.problem_statement) + record = run_container_task( + ContainerTaskConfig( + instance_id=task.instance_id, + condition=args.condition, + repetition=args.repetition, + expected_base_commit=task.base_commit, + task_image=args.task_image, + overlay_image=args.overlay_image, + run_dir=args.run_dir, + source_root=_ROOT, + auth_source=args.auth, + prompt=prompt, + source_commit=args.source_commit, + timeout_seconds=args.timeout_seconds, + ) + ) + print(json.dumps(record, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/tasks.json b/benchmark/tasks.json new file mode 100644 index 0000000..e86740d --- /dev/null +++ b/benchmark/tasks.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "dataset": "princeton-nlp/SWE-bench_Lite", + "split": "dev", + "selection_rule": "three lexicographically lowest SHA-256(instance_id) digests", + "task_set_sha256": "1c8c06935484fd57f4c27c363418884542bcfb06c69d02f480b61fe7687b58eb", + "repetitions": 1, + "conditions": [ + "baseline", + "marginal" + ], + "task_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "task_details": [ + { + "instance_id": "pvlib__pvlib-python-1072", + "repo": "pvlib/pvlib-python", + "base_commit": "04a523fafbd61bc2e49420963b84ed8e2bd1b3cf", + "problem_statement_sha256": "f07d27d56ebb8157d446fe705e1e4c61fc9de31cffb1ebddc2d0b4d1baadc254", + "hints_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "instance_id": "pvlib__pvlib-python-1707", + "repo": "pvlib/pvlib-python", + "base_commit": "40e9e978c170bdde4eeee1547729417665dbc34c", + "problem_statement_sha256": "7c02516a72352ed042e9a91acdf852617b5ed76813b5bf01dd7be21354d08ee5", + "hints_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "instance_id": "pylint-dev__astroid-1978", + "repo": "pylint-dev/astroid", + "base_commit": "0c9ab0fe56703fa83c73e514a1020d398d23fa7f", + "problem_statement_sha256": "51ad593217e29a0c7ff27d7440ba5f33770f11dc1faa2d52cca73f137694bcc3", + "hints_text_sha256": "38b86e0724c5741dab9a455fcbecabb9f1cff161c3f3a706c486b5bb16a14a79" + } + ] +} diff --git a/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md b/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md new file mode 100644 index 0000000..d524e5b --- /dev/null +++ b/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md @@ -0,0 +1,48 @@ +# Measured public benchmark comparison + +This report compares matched executions. It does not estimate or impute missing runs. +MARGINAL overhead is counted in net efficiency and net savings. + +| Metric | Baseline | MARGINAL | Change | +|---|---:|---:|---:| +| Resolved | 0/3 | 0/3 | +0.00 pp | +| Agent tokens | 1,098,747 | 824,839 | 24.93% fewer | +| Effective tokens (incl. governance) | 1,098,747 | 824,839 | 24.93% fewer | +| Effective USD | $0.0000 | $0.0000 | 0.00% lower | +| Effective latency | 593,106 ms | 565,768 ms | 4.61% lower | +| Tool calls | 33 | 32 | 3.03% fewer | +| Repeated calls | 0 | 0 | 0.00% fewer | +| Tokens per resolved task | n/a | n/a | — | +| USD per resolved task | n/a | n/a | — | + +## Governance tax + +MARGINAL overhead: **0 tokens**, **$0.000000**, **7,058 ms**. +Gross agent-token savings: **24.93%**. Net token savings after governance: **24.93%**. + +## Quality and intervention decision + +Net token savings 95.0% bootstrap interval: **21.52% to 28.07%**. +Quality preserved within the 1.00 pp non-inferiority margin: **True**. +Regressions: **0**. Recoveries: **0**. +Reviewed deny recommendations: **0**. False stops: **0** (n/a). +Intervention status: **pass_through**. +No verified successful task was observed, so token efficiency per resolved task is undefined and the intervention cannot be classified as supported. + +`pass_through` is a valid result: it means MARGINAL did not demonstrate enough net value to justify intervention under the preregistered threshold. + +## Scope and interpretation + +This is an **exploratory 3-task smoke with one paired run per task**, not a population estimate or headline performance claim. Codex CLI 0.147.0 and GPT-5.6-sol ran under identical prompts, limits, task order, official task images and base commits. MARGINAL added 7,058 ms of measured local governance latency, zero external governance tokens and zero governance USD. + +The token interval above is only a bootstrap over three task pairs; it cannot estimate run-to-run model variance because there was one repetition. All three observed ON trajectories had zero applied denies. The token difference therefore cannot be attributed to a stop decision, and neither lane's 0/3 resolve rate permits an efficiency-per-success claim. + +## Verification chain + +- Authoritative correctness: SWE-bench 4.1.0 Docker harness, x86_64, official pinned task images; 3/3 completed and 0 infrastructure errors in each lane. +- Cloud audit: [GitHub Actions + Modal run 31474500980](https://github.com/SignalLayerLabs/Marginal/actions/runs/31474500980); 2/3 completed and the same Modal image-build error occurred for `pylint-dev__astroid-1978` in both lanes. The cloud error is retained but excluded from scoring. +- Evidence: [`evidence/smoke-2026-08-11-dbce533/`](evidence/smoke-2026-08-11-dbce533/) contains predictions, telemetry, verifier reports, merged rows, provenance and raw public-evaluator output. +- Frozen task-set SHA-256: `1c8c06935484fd57f4c27c363418884542bcfb06c69d02f480b61fe7687b58eb`. +- MARGINAL source commit used by the runtime overlays: `dbce533e5c8e8def62b0543853d07cfab2bc79c1`. + +The next promotion gate is a preregistered repeated canary with verified successes and enough repetitions to measure trajectory variance. Until then, `pass_through` is the evidence-backed product decision. diff --git a/benchmarks/swebench_lite/README.md b/benchmarks/swebench_lite/README.md index f1ca84d..c6acef9 100644 --- a/benchmarks/swebench_lite/README.md +++ b/benchmarks/swebench_lite/README.md @@ -115,8 +115,14 @@ GitHub Actions authenticates to Modal through the repository secrets `MODAL_TOKE The workflow runs the official SWE-bench evaluator with `--split dev --modal true` for both lanes, merges verifier outcomes into telemetry, runs `marginal public-eval`, adds the Markdown comparison to the Actions job summary, and uploads the complete evidence bundle as an artifact. -The verifier adapter accepts both SWE-bench result layouts currently encountered in the public harness: per-instance `instance_results.jsonl` and the aggregate schema-v2 `..json` report containing `submitted_ids` / `resolved_ids`. Submitted tasks that are not listed as resolved — including empty patches and evaluation errors — remain unresolved rather than being dropped. +The verifier adapter accepts both SWE-bench result layouts currently encountered in the public harness: per-instance `instance_results.jsonl` and the aggregate schema-v2 `..json` report containing `submitted_ids` / `resolved_ids`. Empty patches remain unresolved. `error_ids` and `incomplete_ids` fail closed: infrastructure failures cannot be silently scored as model failures. ## Interpretation A valid run can conclude `supported`, `pass_through`, `quality_regression`, or `false_stop_risk`. A lower token count is not considered a win if verified quality regresses or MARGINAL's own governance overhead removes the apparent saving. + +## First completed smoke + +The evidence bundle in [`evidence/smoke-2026-08-11-dbce533/`](evidence/smoke-2026-08-11-dbce533/) contains the first matched Codex OFF/ON run. The authoritative SWE-bench 4.1 Docker verifier completed 3/3 tasks per lane with zero infrastructure errors; both lanes resolved 0/3. The resulting intervention status is `pass_through`, even though ON used 24.93% fewer measured tokens, because no successful task exists from which to calculate effective compute per resolved task. + +The associated [Modal workflow run](https://github.com/SignalLayerLabs/Marginal/actions/runs/31474500980) completed but reported a symmetric build failure for one task in each lane. Those reports are retained as `verifier_modal_*.json` for audit and are not used as correctness evidence. See `verification.json` for the verifier chain and report digests. diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md new file mode 100644 index 0000000..ed56415 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md @@ -0,0 +1,32 @@ +# Measured public benchmark comparison + +This report compares matched executions. It does not estimate or impute missing runs. +MARGINAL overhead is counted in net efficiency and net savings. + +| Metric | Baseline | MARGINAL | Change | +|---|---:|---:|---:| +| Resolved | 0/3 | 0/3 | +0.00 pp | +| Agent tokens | 1,098,747 | 824,839 | 24.93% fewer | +| Effective tokens (incl. governance) | 1,098,747 | 824,839 | 24.93% fewer | +| Effective USD | $0.0000 | $0.0000 | 0.00% lower | +| Effective latency | 593,106 ms | 565,768 ms | 4.61% lower | +| Tool calls | 33 | 32 | 3.03% fewer | +| Repeated calls | 0 | 0 | 0.00% fewer | +| Tokens per resolved task | n/a | n/a | — | +| USD per resolved task | n/a | n/a | — | + +## Governance tax + +MARGINAL overhead: **0 tokens**, **$0.000000**, **7,058 ms**. +Gross agent-token savings: **24.93%**. Net token savings after governance: **24.93%**. + +## Quality and intervention decision + +Net token savings 95.0% bootstrap interval: **21.52% to 28.07%**. +Quality preserved within the 1.00 pp non-inferiority margin: **True**. +Regressions: **0**. Recoveries: **0**. +Reviewed deny recommendations: **0**. False stops: **0** (n/a). +Intervention status: **pass_through**. +No verified successful task was observed, so token efficiency per resolved task is undefined and the intervention cannot be classified as supported. + +`pass_through` is a valid result: it means MARGINAL did not demonstrate enough net value to justify intervention under the preregistered threshold. diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson new file mode 100644 index 0000000..75a79b3 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson @@ -0,0 +1,3 @@ +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":177424,"repeated_calls":0,"reviewed_stops":0,"tokens":370497,"tool_calls":11,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":268283,"repeated_calls":0,"reviewed_stops":0,"tokens":383851,"tool_calls":12,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":147399,"repeated_calls":0,"reviewed_stops":0,"tokens":344399,"tool_calls":10,"usd":0.0} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_predictions.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_predictions.ndjson new file mode 100644 index 0000000..c7852e4 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_predictions.ndjson @@ -0,0 +1,3 @@ +{"instance_id":"pvlib__pvlib-python-1072","model_name_or_path":"gpt-5.6-sol","model_patch":"diff --git a/pvlib/temperature.py b/pvlib/temperature.py\nindex 1d98736..e87efbe 100644\n--- a/pvlib/temperature.py\n+++ b/pvlib/temperature.py\n@@ -599,7 +599,8 @@ def fuentes(poa_global, temp_air, wind_speed, noct_installed, module_height=5,\n # n.b. the way Fuentes calculates the first timedelta makes it seem like\n # the value doesn't matter -- rather than recreate it here, just assume\n # it's the same as the second timedelta:\n- timedelta_hours = np.diff(poa_global.index).astype(float) / 1e9 / 60 / 60\n+ index = poa_global.index\n+ timedelta_hours = (index[1:] - index[:-1]).total_seconds() / 60 / 60\n timedelta_hours = np.append([timedelta_hours[0]], timedelta_hours)\n \n tamb_array = temp_air + 273.15\ndiff --git a/pvlib/tests/test_temperature.py b/pvlib/tests/test_temperature.py\nindex 245c5d2..66d8a3b 100644\n--- a/pvlib/tests/test_temperature.py\n+++ b/pvlib/tests/test_temperature.py\n@@ -190,3 +190,22 @@ def test_fuentes(filename, inoct):\n night_difference = expected_tcell[is_night] - actual_tcell[is_night]\n assert night_difference.max() < 6\n assert night_difference.min() > 0\n+\n+\n+def test_fuentes_tz_aware_index():\n+ index = pd.date_range('2019-01-01', freq='h', periods=3)\n+ inputs = {\n+ 'poa_global': pd.Series(1000, index=index),\n+ 'temp_air': pd.Series(20, index=index),\n+ 'wind_speed': pd.Series(1, index=index),\n+ 'noct_installed': 45,\n+ }\n+ expected = temperature.fuentes(**inputs)\n+\n+ aware_index = index.tz_localize('UTC')\n+ for name in ('poa_global', 'temp_air', 'wind_speed'):\n+ inputs[name].index = aware_index\n+ expected.index = aware_index\n+\n+ actual = temperature.fuentes(**inputs)\n+ assert_series_equal(expected, actual)\n"} +{"instance_id":"pvlib__pvlib-python-1707","model_name_or_path":"gpt-5.6-sol","model_patch":"diff --git a/pvlib/iam.py b/pvlib/iam.py\nindex 3eaa6b4..283d613 100644\n--- a/pvlib/iam.py\n+++ b/pvlib/iam.py\n@@ -163,7 +163,9 @@ def physical(aoi, n=1.526, K=4.0, L=0.002, *, n_ar=None):\n n2 = n_ar\n \n # incidence angle\n- costheta = np.maximum(0, cosd(aoi)) # always >= 0\n+ aoi_gte_90 = np.abs(aoi) >= 90\n+ # use a safe value for angles whose result will be set to zero\n+ costheta = np.where(aoi_gte_90, 1, np.maximum(0, cosd(aoi)))\n sintheta = np.sqrt(1 - costheta**2) # always >= 0\n n1costheta1 = n1 * costheta\n n2costheta1 = n2 * costheta\n@@ -215,6 +217,12 @@ def physical(aoi, n=1.526, K=4.0, L=0.002, *, n_ar=None):\n # incidence angle modifier\n iam = (tau_s + tau_p) / 2 / tau_0\n \n+ # for light coming from behind the plane, none can enter the module\n+ iam = np.where(aoi_gte_90, 0, iam)\n+\n+ if isinstance(aoi, pd.Series):\n+ iam = pd.Series(iam, index=aoi.index)\n+\n return iam\n \n \ndiff --git a/pvlib/tests/test_iam.py b/pvlib/tests/test_iam.py\nindex eba1c66..fb31966 100644\n--- a/pvlib/tests/test_iam.py\n+++ b/pvlib/tests/test_iam.py\n@@ -72,6 +72,8 @@ def test_physical_scalar():\n iam = _iam.physical(aoi, 1.526, 0.002, 4)\n expected = 0.98797788\n assert_allclose(iam, expected, equal_nan=True)\n+ iam = _iam.physical(aoi=100, n=1)\n+ assert_allclose(iam, 0)\n aoi = np.nan\n iam = _iam.physical(aoi, 1.526, 0.002, 4)\n expected = np.nan\n"} +{"instance_id":"pylint-dev__astroid-1978","model_name_or_path":"gpt-5.6-sol","model_patch":"diff --git a/astroid/raw_building.py b/astroid/raw_building.py\nindex cc3aa015..cff66437 100644\n--- a/astroid/raw_building.py\n+++ b/astroid/raw_building.py\n@@ -471,7 +471,9 @@ class InspectBuilder:\n # check if it sounds valid and then add an import node, else use a\n # dummy node\n try:\n- getattr(sys.modules[modname], name)\n+ with warnings.catch_warnings():\n+ warnings.simplefilter(\"ignore\")\n+ getattr(sys.modules[modname], name)\n except (KeyError, AttributeError):\n attach_dummy_node(node, name, member)\n else:\ndiff --git a/tests/unittest_raw_building.py b/tests/unittest_raw_building.py\nindex 64c22727..ae183d50 100644\n--- a/tests/unittest_raw_building.py\n+++ b/tests/unittest_raw_building.py\n@@ -8,8 +8,11 @@ to simulate issues in unittest below\n # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE\n # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt\n \n+import sys\n import types\n import unittest\n+import warnings\n+from unittest import mock\n \n import _io\n import pytest\n@@ -108,6 +111,27 @@ class RawBuildingTC(unittest.TestCase):\n # This should not raise an exception\n AstroidBuilder().module_build(m, \"test\")\n \n+ def test_imported_member_deprecation(self) -> None:\n+ imported_module = types.ModuleType(\"imported_module\")\n+ imported_member = type(\n+ \"deprecated_member\", (), {\"__module__\": imported_module.__name__}\n+ )\n+\n+ def deprecated_getattr(name):\n+ if name == \"deprecated_member\":\n+ warnings.warn(\"This attribute is deprecated\", FutureWarning)\n+ return imported_member\n+ raise AttributeError(name)\n+\n+ imported_module.__getattr__ = deprecated_getattr\n+ module = types.ModuleType(\"test\")\n+ module.deprecated_member = imported_member\n+\n+ with mock.patch.dict(sys.modules, {imported_module.__name__: imported_module}):\n+ with warnings.catch_warnings():\n+ warnings.simplefilter(\"error\", FutureWarning)\n+ AstroidBuilder().inspect_build(module)\n+\n def test_module_object_with_broken_getattr(self) -> None:\n # Tests https://github.com/PyCQA/astroid/issues/1958\n # When astroid deep inspection of modules raises\n"} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson new file mode 100644 index 0000000..6e43e68 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson @@ -0,0 +1,3 @@ +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":177424,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":370497,"tool_calls":11,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":268283,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":383851,"tool_calls":12,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":147399,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":344399,"tool_calls":10,"usd":0.0} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/manifest.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/manifest.json new file mode 100644 index 0000000..da61b8d --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/manifest.json @@ -0,0 +1,22 @@ +{ + "agent": "codex-cli", + "agent_version": "0.147.0", + "benchmark": "swe-bench-lite-dev-canary", + "dataset": "princeton-nlp/SWE-bench_Lite", + "limits": { + "repetitions": 1, + "timeout_seconds": 1800 + }, + "marginal": { + "commit": "dbce533e5c8e8def62b0543853d07cfab2bc79c1", + "mode": "enforce", + "policy": "balanced+diminishing-defaults", + "version": "0.2.0+unreleased" + }, + "model": "gpt-5.6-sol", + "prompt_sha256": "3a2ced1afb7e9a74176090db264259c52b6ebcf383d787a1030f83ecb6d67416", + "schema_version": 1, + "split": "dev", + "task_set": "smoke", + "task_set_sha256": "1c8c06935484fd57f4c27c363418884542bcfb06c69d02f480b61fe7687b58eb" +} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson new file mode 100644 index 0000000..d321ce7 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson @@ -0,0 +1,3 @@ +{"applied_denies":0,"false_stops":0,"governance_latency_ms":1823,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":178765,"repeated_calls":0,"reviewed_stops":0,"tokens":275858,"tool_calls":11,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":2224,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":236330,"repeated_calls":0,"reviewed_stops":0,"tokens":301257,"tool_calls":13,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":3011,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":143615,"repeated_calls":0,"reviewed_stops":0,"tokens":247724,"tool_calls":8,"usd":0.0} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_predictions.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_predictions.ndjson new file mode 100644 index 0000000..774b75a --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_predictions.ndjson @@ -0,0 +1,3 @@ +{"instance_id":"pvlib__pvlib-python-1072","model_name_or_path":"gpt-5.6-sol","model_patch":"diff --git a/pvlib/temperature.py b/pvlib/temperature.py\nindex 1d98736..787693a 100644\n--- a/pvlib/temperature.py\n+++ b/pvlib/temperature.py\n@@ -599,7 +599,9 @@ def fuentes(poa_global, temp_air, wind_speed, noct_installed, module_height=5,\n # n.b. the way Fuentes calculates the first timedelta makes it seem like\n # the value doesn't matter -- rather than recreate it here, just assume\n # it's the same as the second timedelta:\n- timedelta_hours = np.diff(poa_global.index).astype(float) / 1e9 / 60 / 60\n+ timedelta_hours = (\n+ poa_global.index[1:] - poa_global.index[:-1]\n+ ).total_seconds() / 60 / 60\n timedelta_hours = np.append([timedelta_hours[0]], timedelta_hours)\n \n tamb_array = temp_air + 273.15\ndiff --git a/pvlib/tests/test_temperature.py b/pvlib/tests/test_temperature.py\nindex 245c5d2..95c7335 100644\n--- a/pvlib/tests/test_temperature.py\n+++ b/pvlib/tests/test_temperature.py\n@@ -190,3 +190,22 @@ def test_fuentes(filename, inoct):\n night_difference = expected_tcell[is_night] - actual_tcell[is_night]\n assert night_difference.max() < 6\n assert night_difference.min() > 0\n+\n+\n+def test_fuentes_tz_aware():\n+ index = pd.date_range('2019-01-01', freq='h', periods=3)\n+ inputs = {\n+ 'poa_global': pd.Series(1000, index=index),\n+ 'temp_air': pd.Series(20, index=index),\n+ 'wind_speed': pd.Series(1, index=index),\n+ 'noct_installed': 45,\n+ }\n+ expected = temperature.fuentes(**inputs)\n+\n+ aware_index = index.tz_localize('UTC')\n+ for name in ('poa_global', 'temp_air', 'wind_speed'):\n+ inputs[name].index = aware_index\n+ expected.index = aware_index\n+\n+ actual = temperature.fuentes(**inputs)\n+ assert_series_equal(expected, actual)\n"} +{"instance_id":"pvlib__pvlib-python-1707","model_name_or_path":"gpt-5.6-sol","model_patch":"diff --git a/pvlib/iam.py b/pvlib/iam.py\nindex 3eaa6b4..e4d983a 100644\n--- a/pvlib/iam.py\n+++ b/pvlib/iam.py\n@@ -155,6 +155,8 @@ def physical(aoi, n=1.526, K=4.0, L=0.002, *, n_ar=None):\n pvlib.iam.interp\n pvlib.iam.sapm\n \"\"\"\n+ aoi_input = aoi\n+ aoi_gte_90 = np.abs(aoi) >= 90\n n1, n3 = 1, n\n if n_ar is None or np.allclose(n_ar, n1):\n # no AR coating\n@@ -163,7 +165,9 @@ def physical(aoi, n=1.526, K=4.0, L=0.002, *, n_ar=None):\n n2 = n_ar\n \n # incidence angle\n- costheta = np.maximum(0, cosd(aoi)) # always >= 0\n+ # use a non-singular value during the calculation for angles whose final\n+ # result is known to be zero\n+ costheta = np.where(aoi_gte_90, 1, np.maximum(0, cosd(aoi)))\n sintheta = np.sqrt(1 - costheta**2) # always >= 0\n n1costheta1 = n1 * costheta\n n2costheta1 = n2 * costheta\n@@ -215,6 +219,13 @@ def physical(aoi, n=1.526, K=4.0, L=0.002, *, n_ar=None):\n # incidence angle modifier\n iam = (tau_s + tau_p) / 2 / tau_0\n \n+ # no light can enter through the front surface at or beyond grazing\n+ # incidence\n+ iam = np.where(aoi_gte_90, 0, iam)\n+\n+ if isinstance(aoi_input, pd.Series):\n+ iam = pd.Series(iam, index=aoi_input.index)\n+\n return iam\n \n \ndiff --git a/pvlib/tests/test_iam.py b/pvlib/tests/test_iam.py\nindex eba1c66..acb7122 100644\n--- a/pvlib/tests/test_iam.py\n+++ b/pvlib/tests/test_iam.py\n@@ -78,6 +78,11 @@ def test_physical_scalar():\n assert_allclose(iam, expected, equal_nan=True)\n \n \n+def test_physical_no_reflection_behind_module():\n+ iam = _iam.physical(aoi=100, n=1)\n+ assert_allclose(iam, 0)\n+\n+\n def test_martin_ruiz():\n \n aoi = 45.\n"} +{"instance_id":"pylint-dev__astroid-1978","model_name_or_path":"gpt-5.6-sol","model_patch":"diff --git a/astroid/raw_building.py b/astroid/raw_building.py\nindex cc3aa015..cff66437 100644\n--- a/astroid/raw_building.py\n+++ b/astroid/raw_building.py\n@@ -471,7 +471,9 @@ class InspectBuilder:\n # check if it sounds valid and then add an import node, else use a\n # dummy node\n try:\n- getattr(sys.modules[modname], name)\n+ with warnings.catch_warnings():\n+ warnings.simplefilter(\"ignore\")\n+ getattr(sys.modules[modname], name)\n except (KeyError, AttributeError):\n attach_dummy_node(node, name, member)\n else:\ndiff --git a/tests/unittest_raw_building.py b/tests/unittest_raw_building.py\nindex 64c22727..8b130d59 100644\n--- a/tests/unittest_raw_building.py\n+++ b/tests/unittest_raw_building.py\n@@ -108,6 +108,19 @@ class RawBuildingTC(unittest.TestCase):\n # This should not raise an exception\n AstroidBuilder().module_build(m, \"test\")\n \n+ def test_build_imported_member_deprecation(self) -> None:\n+ m = types.ModuleType(\"test\")\n+\n+ class Float64Index:\n+ pass\n+\n+ Float64Index.__module__ = fm.__name__\n+ m.Float64Index = Float64Index\n+\n+ # Looking up the imported member in its defining module should not\n+ # expose warnings raised by the module's custom __getattr__.\n+ AstroidBuilder().module_build(m, \"test\")\n+\n def test_module_object_with_broken_getattr(self) -> None:\n # Tests https://github.com/PyCQA/astroid/issues/1958\n # When astroid deep inspection of modules raises\n"} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson new file mode 100644 index 0000000..9b93096 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson @@ -0,0 +1,3 @@ +{"applied_denies":0,"false_stops":0,"governance_latency_ms":1823,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":178765,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":275858,"tool_calls":11,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":2224,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":236330,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":301257,"tool_calls":13,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":3011,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":143615,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":247724,"tool_calls":8,"usd":0.0} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/provenance.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/provenance.json new file mode 100644 index 0000000..105af0e --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/provenance.json @@ -0,0 +1,28 @@ +{ + "tasks": [ + { + "base_commit": "04a523fafbd61bc2e49420963b84ed8e2bd1b3cf", + "baseline_overlay_image": "sha256:d32a981a154d51043281e301e4a9ccf2fdecf310114fbe0bb8f8969b03549a40", + "instance_id": "pvlib__pvlib-python-1072", + "marginal_overlay_image": "sha256:d32a981a154d51043281e301e4a9ccf2fdecf310114fbe0bb8f8969b03549a40", + "prompt_sha256": "6a4885b85b49ded91db260fe34775f577ce3b3127c4d9ab8ec05611a499f4696", + "task_image": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1072@sha256:c8e43bebd10d7d2330820af520361dc6adc9642f98c7b7b7c415cca39852ffdd" + }, + { + "base_commit": "40e9e978c170bdde4eeee1547729417665dbc34c", + "baseline_overlay_image": "sha256:859a3989c6cd8aa428b2e3b0f38b374049a97379ae9861dc8554baeaf228b227", + "instance_id": "pvlib__pvlib-python-1707", + "marginal_overlay_image": "sha256:859a3989c6cd8aa428b2e3b0f38b374049a97379ae9861dc8554baeaf228b227", + "prompt_sha256": "b7691aba45ae5d3dfbfeb1e372a4b4b61c0ed122cda3bc55a2218aaadafd2a02", + "task_image": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1707@sha256:3c103594eb39c0564cfd71f12ecca0d75a9a4a821f33c6e43b036d2271e75de7" + }, + { + "base_commit": "0c9ab0fe56703fa83c73e514a1020d398d23fa7f", + "baseline_overlay_image": "sha256:cfdce1645338601413efcc6453d0d6336a20a5810de721e05a25ee18042ff82c", + "instance_id": "pylint-dev__astroid-1978", + "marginal_overlay_image": "sha256:cfdce1645338601413efcc6453d0d6336a20a5810de721e05a25ee18042ff82c", + "prompt_sha256": "eade552f513e57f3dfad7cb84ac00ac45d83f4e168201f8c86e554e9278f9f4a", + "task_image": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978@sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685" + } + ] +} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json new file mode 100644 index 0000000..8c048dd --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json @@ -0,0 +1 @@ +{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": 0.0}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": 0.0}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": 0.0}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "quality": {"false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": true, "preserved_within_one_pp": true, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "tasks": 3} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/run-records.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/run-records.ndjson new file mode 100644 index 0000000..3ae2886 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/run-records.ndjson @@ -0,0 +1,6 @@ +{"condition":"baseline","configuration_sha256":"2a93efa13d62d485d87f5f4d028062a23e51dd4997f857f0bda3a6817b5d6f8f","diff_lines":22,"error_code":null,"file_operations":1,"files_modified":2,"governance":{"latency_ms":0.0,"tokens":0,"usd":0.0},"instance_id":"pvlib__pvlib-python-1072","interventions":{"applied_denies":0,"false_stops":0,"recommended_denies":0,"reviewed":0},"model_reroutes":null,"patch_sha256":"333368e126316a6c1a98f554ffd4b60e5655a2493fb7ffecb1baaeb8ffc7c0de","repeated_calls":0,"repetition":1,"resolved":null,"run_status":"completed","schema_version":1,"searches":1,"shell_commands":10,"test_executions":0,"tokens":{"cached_input":317184,"input":366992,"output":3505,"reasoning":1540,"total":370497},"tool_calls":11,"wall_time_ms":177424} +{"condition":"marginal","configuration_sha256":"e086c14463a0b340f2b0216d3a11c41121bed69c2b8016e1cfa729f8e989b86d","diff_lines":23,"error_code":null,"file_operations":1,"files_modified":2,"governance":{"latency_ms":1822.778462,"tokens":0,"usd":0.0},"instance_id":"pvlib__pvlib-python-1072","interventions":{"applied_denies":0,"false_stops":0,"recommended_denies":0,"reviewed":0},"model_reroutes":null,"patch_sha256":"c9cf4cd9c301d2c162c1fb3c83e2993dee498e25ecbf2fc1f95175ae59680c28","repeated_calls":0,"repetition":1,"resolved":null,"run_status":"completed","schema_version":1,"searches":2,"shell_commands":10,"test_executions":0,"tokens":{"cached_input":241152,"input":272368,"output":3490,"reasoning":1467,"total":275858},"tool_calls":11,"wall_time_ms":178765} +{"condition":"baseline","configuration_sha256":"e34dbe0de55981b21854a24912aabe972e1424c77154d8e21990266b0d2d4d43","diff_lines":12,"error_code":null,"file_operations":2,"files_modified":2,"governance":{"latency_ms":0.0,"tokens":0,"usd":0.0},"instance_id":"pvlib__pvlib-python-1707","interventions":{"applied_denies":0,"false_stops":0,"recommended_denies":0,"reviewed":0},"model_reroutes":null,"patch_sha256":"a00908b3a08bac84efd7bce6d5f4fcbe2b070d8753a0df8aa52c1b179fcf0815","repeated_calls":0,"repetition":1,"resolved":null,"run_status":"completed","schema_version":1,"searches":2,"shell_commands":10,"test_executions":0,"tokens":{"cached_input":331776,"input":377314,"output":6537,"reasoning":4361,"total":383851},"tool_calls":12,"wall_time_ms":268283} +{"condition":"marginal","configuration_sha256":"e0344af829ec67ca716b689f0f2d46c44c205018f66dea7ca7fa808b58e97a90","diff_lines":18,"error_code":null,"file_operations":2,"files_modified":2,"governance":{"latency_ms":2223.662921,"tokens":0,"usd":0.0},"instance_id":"pvlib__pvlib-python-1707","interventions":{"applied_denies":0,"false_stops":0,"recommended_denies":0,"reviewed":0},"model_reroutes":null,"patch_sha256":"ae0855a047422e4dcf96748cedd0eef358ad2466ad26515dc2bc642a7c2c518c","repeated_calls":0,"repetition":1,"resolved":null,"run_status":"completed","schema_version":1,"searches":2,"shell_commands":11,"test_executions":1,"tokens":{"cached_input":267520,"input":296446,"output":4811,"reasoning":2553,"total":301257},"tool_calls":13,"wall_time_ms":236330} +{"condition":"baseline","configuration_sha256":"00bd62dd9a7d088f85b40c4927ec9a962db9241f6ca735b7d1ef557e1dd3cbe8","diff_lines":28,"error_code":null,"file_operations":1,"files_modified":2,"governance":{"latency_ms":0.0,"tokens":0,"usd":0.0},"instance_id":"pylint-dev__astroid-1978","interventions":{"applied_denies":0,"false_stops":0,"recommended_denies":0,"reviewed":0},"model_reroutes":null,"patch_sha256":"54ad66d28f1189192f7b535f7c3c3a46620e8d9b09b4671bba4c877866c28368","repeated_calls":0,"repetition":1,"resolved":null,"run_status":"completed","schema_version":1,"searches":4,"shell_commands":9,"test_executions":1,"tokens":{"cached_input":289280,"input":341044,"output":3355,"reasoning":1440,"total":344399},"tool_calls":10,"wall_time_ms":147399} +{"condition":"marginal","configuration_sha256":"ecc777fcbe6264db1482736611e2207c900f2e2f170d5a37cfd6785aa090cb64","diff_lines":17,"error_code":null,"file_operations":2,"files_modified":2,"governance":{"latency_ms":3011.415667,"tokens":0,"usd":0.0},"instance_id":"pylint-dev__astroid-1978","interventions":{"applied_denies":0,"false_stops":0,"recommended_denies":0,"reviewed":0},"model_reroutes":null,"patch_sha256":"fd8183a1cafdd0f48ccd77e1b9fd19215eb8d35574751aa1b2d6ab31e4b9fb59","repeated_calls":0,"repetition":1,"resolved":null,"run_status":"completed","schema_version":1,"searches":2,"shell_commands":6,"test_executions":0,"tokens":{"cached_input":204544,"input":245207,"output":2517,"reasoning":1349,"total":247724},"tool_calls":8,"wall_time_ms":143615} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verification.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verification.json new file mode 100644 index 0000000..8cfe5a4 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verification.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "task_set": "smoke", + "authoritative": { + "backend": "official-swebench-docker", + "swebench_version": "4.1.0", + "architecture": "x86_64", + "baseline": { + "run_id": "local-smoke-baseline-31474500980", + "submitted": 3, + "completed": 3, + "resolved": 0, + "errors": 0, + "report": "verifier_baseline.json", + "report_sha256": "5e7fc0deab6548bc7b8e487aaf01389e19f4a27df880a9d97cccafac3e23c412" + }, + "marginal": { + "run_id": "local-smoke-on-31474500980", + "submitted": 3, + "completed": 3, + "resolved": 0, + "errors": 0, + "report": "verifier_marginal.json", + "report_sha256": "5e7fc0deab6548bc7b8e487aaf01389e19f4a27df880a9d97cccafac3e23c412" + } + }, + "modal_audit": { + "workflow_run": "https://github.com/SignalLayerLabs/Marginal/actions/runs/31474500980", + "modal_version": "1.5.3", + "swebench_version": "4.1.0", + "status": "infrastructure_error_not_scored", + "baseline_completed": 2, + "marginal_completed": 2, + "baseline_errors": 1, + "marginal_errors": 1, + "symmetric_error_ids": [ + "pylint-dev__astroid-1978" + ], + "baseline_report": "verifier_modal_baseline.json", + "marginal_report": "verifier_modal_marginal.json", + "report_sha256": "9c7ebafff8e259de16a5cad8f20dc1ad04553a12e0e3960175b7ac207e503e4d" + }, + "public_result": { + "resolved": "0/3 -> 0/3", + "net_token_savings_percent": 24.93, + "intervention_status": "pass_through", + "eligible_for_support": false + } +} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_baseline.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_baseline.json new file mode 100644 index 0000000..de7403b --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_baseline.json @@ -0,0 +1,29 @@ +{ + "total_instances": 3, + "submitted_instances": 3, + "completed_instances": 3, + "resolved_instances": 0, + "unresolved_instances": 3, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "resolved_ids": [], + "unresolved_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "error_ids": [], + "schema_version": 2 +} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_marginal.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_marginal.json new file mode 100644 index 0000000..de7403b --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_marginal.json @@ -0,0 +1,29 @@ +{ + "total_instances": 3, + "submitted_instances": 3, + "completed_instances": 3, + "resolved_instances": 0, + "unresolved_instances": 3, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "resolved_ids": [], + "unresolved_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "error_ids": [], + "schema_version": 2 +} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_modal_baseline.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_modal_baseline.json new file mode 100644 index 0000000..0bd1789 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_modal_baseline.json @@ -0,0 +1,32 @@ +{ + "total_instances": 3, + "submitted_instances": 3, + "completed_instances": 2, + "resolved_instances": 0, + "unresolved_instances": 2, + "empty_patch_instances": 0, + "error_instances": 1, + "completed_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "resolved_ids": [], + "unresolved_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707" + ], + "error_ids": [ + "pylint-dev__astroid-1978" + ], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [] +} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_modal_marginal.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_modal_marginal.json new file mode 100644 index 0000000..0bd1789 --- /dev/null +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/verifier_modal_marginal.json @@ -0,0 +1,32 @@ +{ + "total_instances": 3, + "submitted_instances": 3, + "completed_instances": 2, + "resolved_instances": 0, + "unresolved_instances": 2, + "empty_patch_instances": 0, + "error_instances": 1, + "completed_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707", + "pylint-dev__astroid-1978" + ], + "resolved_ids": [], + "unresolved_ids": [ + "pvlib__pvlib-python-1072", + "pvlib__pvlib-python-1707" + ], + "error_ids": [ + "pylint-dev__astroid-1978" + ], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [] +} diff --git a/benchmarks/swebench_lite/merge_results.py b/benchmarks/swebench_lite/merge_results.py index 2936d58..3adb36c 100644 --- a/benchmarks/swebench_lite/merge_results.py +++ b/benchmarks/swebench_lite/merge_results.py @@ -107,6 +107,20 @@ def _index_aggregate_report(report: dict[str, Any]) -> dict[str, bool]: resolved_set = set(resolved) if not resolved_set <= submitted_set: raise MergeError("aggregate resolved_ids must be a subset of submitted_ids") + infrastructure_failures: list[str] = [] + for key in ("error_ids", "incomplete_ids"): + values = report.get(key, []) + if not isinstance(values, list) or not all( + isinstance(item, str) and item for item in values + ): + raise MergeError(f"aggregate {key} must contain non-empty strings") + infrastructure_failures.extend(values) + if infrastructure_failures: + failed = sorted(set(infrastructure_failures)) + raise MergeError( + "verifier infrastructure failures cannot be scored as unresolved: " + + ", ".join(failed[:5]) + ) return {instance_id: instance_id in resolved_set for instance_id in submitted} diff --git a/benchmarks/swebench_lite/public-benchmark.json b/benchmarks/swebench_lite/public-benchmark.json new file mode 100644 index 0000000..8c048dd --- /dev/null +++ b/benchmarks/swebench_lite/public-benchmark.json @@ -0,0 +1 @@ +{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": 0.0}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": 0.0}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": 0.0}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "quality": {"false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": true, "preserved_within_one_pp": true, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "tasks": 3} diff --git a/docs/superpowers/plans/2026-08-11-codex-benchmark-integration.md b/docs/superpowers/plans/2026-08-11-codex-benchmark-integration.md new file mode 100644 index 0000000..5b2804c --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-codex-benchmark-integration.md @@ -0,0 +1,202 @@ +# Codex + MARGINAL End-to-End Benchmark Plan + +> Execute test-first with `superpowers:test-driven-development`, diagnose failures with +> `superpowers:systematic-debugging`, and require fresh evidence under +> `superpowers:verification-before-completion`. + +**Goal:** Produce an independently verifiable paired Codex OFF versus Codex+MARGINAL +benchmark in official SWE-bench environments, publish the exact outcome at the top of the +README and website, and preserve machine-readable reproduction artifacts. + +**Architecture:** A host orchestrator launches pinned Codex inside each official +per-instance SWE-bench image. OFF contains only Codex and telemetry collection; ON adds a +per-run MARGINAL daemon and official hooks. Strict run records feed the repository's +SWE-bench evidence protocol and GitHub Actions/Modal verifier. + +**Stack:** Python 3.10+, pytest, Docker/Colima, SWE-bench, Modal, Codex CLI 0.147.0, +GitHub Actions, static HTML/CSS/JavaScript. + +## Frozen constraints + +- MARGINAL source commit: `4c8856401b4c752d5c214df5e84b9632d9897ec9`. +- Codex CLI/model/effort: `0.147.0`, `gpt-5.6-sol`, `high`. +- Correctness is assigned only by the official SWE-bench verifier. +- No credentials, prompts, task source, or raw tool output enter public artifacts. +- OFF has no MARGINAL process, hook, prompt text, configuration, or state. +- ON failures are explicit and cannot degrade silently to pass-through. +- No policy, prompt, timeout, or task-set tuning after comparative outcomes are observed. +- A token-saving headline requires preserved paired correctness and positive verified data. + +## Task 1: Preserve and re-baseline the existing feasibility adapter + +**Files:** `benchmark/**`, `tests/benchmark/**`, design and plan documents. + +- [ ] Create a dedicated benchmark branch without discarding the feasibility work. +- [ ] Run the current full unit/type/lint suite and record the starting evidence. +- [ ] Commit the reviewed feasibility baseline so later runtime changes remain auditable. +- [ ] Confirm result JSONL files contain no inference outcomes before the design amendment. + +## Task 2: Correct completed-action semantics test-first + +**Files:** +- Modify: `benchmark/codex_adapter/engine.py` +- Modify: `benchmark/codex_adapter/daemon.py` +- Test: `tests/benchmark/test_governance_engine.py` +- Test: `tests/benchmark/test_daemon_protocol.py` + +- [ ] Add a failing test where a shell `PostToolUse` commits evidence and the third exact, + unchanged proposal is denied. +- [ ] Add failing tests for malformed, duplicate, missing, and identity-mismatched post + events and pending actions at shutdown. +- [ ] Observe the intended red failures before editing production code. +- [ ] Implement the frozen completion semantics without inferring command exit status. +- [ ] Confirm exact decision accounting and the targeted tests pass. + +## Task 3: Add official task-image resolution and overlay build test-first + +**Files:** +- Create: `benchmark/codex_adapter/container_runtime.py` +- Create: `benchmark/container/Dockerfile.tools` +- Create: `benchmark/container/entrypoint.sh` +- Test: `tests/benchmark/test_container_runtime.py` + +- [ ] Add failing tests for deterministic official image names/digests, platform, mount + paths, read-only tool layer, model credential injection, and environment allowlist. +- [ ] Add failing tests proving the task worktree/run directory cannot alias and no user + home or auth file is mounted. +- [ ] Implement a subprocess argument builder with no shell interpolation. +- [ ] Build a pinned tool layer containing Linux Codex 0.147.0 and an isolated MARGINAL + environment; record its content digest. +- [ ] Inspect the built image and verify it contains no credentials or task content. +- [ ] Confirm all container-runtime tests pass. + +## Task 4: Integrate container execution into the runner test-first + +**Files:** +- Modify: `benchmark/codex_adapter/runner.py` +- Modify: `benchmark/codex_adapter/preflight.py` +- Modify: `benchmark/scripts/run_codex_task.py` +- Modify: `benchmark/environment.json` +- Test: `tests/benchmark/test_codex_runner.py` +- Test: `tests/benchmark/test_preflight.py` + +- [ ] Add a fake-container failing test proving OFF/ON receive identical runtime inputs and + ON alone receives hooks/daemon configuration. +- [ ] Add failing tests for wrong image digest, unavailable engine, architecture mismatch, + missing model auth, missing sandbox network isolation, and dirty base commit. +- [ ] Implement container lifecycle, timeouts, signal cleanup, bind mounts, raw-event + collection, and patch extraction. +- [ ] Replace the obsolete unconditional preflight block with concrete runtime checks. +- [ ] Require full preflight inside the callable runner, not only the CLI. +- [ ] Confirm targeted runner/preflight tests pass. + +## Task 5: Prove a deterministic container integration fixture + +**Files:** +- Create: `benchmark/fixtures/container_task/**` +- Create: `benchmark/scripts/container_smoke.py` +- Test: `tests/benchmark/test_container_integration.py` + +- [ ] Write a fixture whose fake model stream performs a shell action, repeats it, edits a + file, and emits a final patch. +- [ ] Observe the integration test fail before runtime wiring exists. +- [ ] Run OFF and ON in disposable containers. +- [ ] Prove tool execution occurs inside the container, ON has exact pre/post coverage, + the third same-state repeat is denied, OFF has no MARGINAL footprint, and both patches + are extracted outside `/testbed`. +- [ ] Prove secret scanning rejects credential-shaped fixture output. + +## Task 6: Bridge run records to the existing SWE-bench evidence protocol + +**Files:** +- Create: `benchmark/codex_adapter/evidence.py` +- Create: `benchmark/scripts/export_swebench_evidence.py` +- Modify: `benchmarks/swebench_lite/protocol.py` only if a backward-compatible schema field + is required. +- Test: `tests/benchmark/test_evidence_export.py` + +- [ ] Add failing tests mapping strict run records to baseline/ON predictions and metrics. +- [ ] Prove mismatched task IDs, commits, image digests, configs, failed integrity gates, + duplicate rows, and secret findings abort export. +- [ ] Implement deterministic manifest/config hashes and NDJSON export. +- [ ] Validate output with the existing `validate` protocol command. +- [ ] Confirm legacy canary fixtures remain valid. + +## Task 7: Freeze smoke and canary manifests + +**Files:** +- Modify: `benchmark/tasks.json` +- Create: `benchmark/canary-tasks.json` +- Modify: `benchmark/methodology.md` +- Modify: `benchmark/environment.json` + +- [ ] Preserve the three preregistered smoke task IDs and task-set hash. +- [ ] Select a deterministic twenty-task SWE-bench Lite subset without looking at model + outcomes; record selection rule, IDs, base commits, dataset revision, and SHA-256. +- [ ] Record exact official image digests, overlay digest, host/engine details, timeouts, + prompt/config hashes, and amendment history. +- [ ] Validate manifests and prove no gold patch or test patch enters prompts. + +## Task 8: Fresh local verification before paid inference + +- [ ] Run `ruff format --check .`. +- [ ] Run `ruff check .`. +- [ ] Run `mypy src/marginal benchmark` with the repository's supported configuration. +- [ ] Run `.venv/bin/python -m pytest -p no:cacheprovider -q`. +- [ ] Run the deterministic container integration fixture. +- [ ] Run full benchmark preflight and validate exact tool/image/model availability. +- [ ] Run `git diff --check` and a repository secret scan. +- [ ] Commit and push the implementation branch only after every local gate is green. + +## Task 9: Execute and verify the frozen three-task smoke + +- [ ] Materialize six independent clean runs: three OFF, then three ON. +- [ ] Persist raw private run artifacts and strict public records separately. +- [ ] Validate exact lane pairing and telemetry coverage before verifier spend. +- [ ] Export committed evidence under `benchmarks/swebench_lite/evidence/`. +- [ ] Push evidence and dispatch `.github/workflows/swebench-lite-canary.yml`. +- [ ] Wait for the official Modal verifier and download its artifact. +- [ ] Validate provenance and merge verifier outcomes into paired smoke results. +- [ ] If any integrity gate fails, fix the pipeline and rerun the affected stage from clean + state; do not analyze partial outcomes. + +## Task 10: Execute and verify the frozen twenty-task canary + +- [ ] Promote only after smoke telemetry and official verification are complete. +- [ ] Materialize forty independent clean runs under the unchanged frozen configuration. +- [ ] Export, commit, push, dispatch, and wait for the official Modal verification. +- [ ] Validate and merge the canary artifact without dropping failures. +- [ ] Generate per-task rows, paired outcome counts, token deltas, latency, governance + overhead, intervention counts, and bootstrap confidence intervals where defensible. + +## Task 11: Publish evidence-first README and website + +**Files:** +- Modify: `README.md` +- Modify: `site/index.html` +- Modify: `site/app.js` +- Modify: `site/styles.css` +- Create or update: `benchmarks/swebench_lite/PUBLIC_BENCHMARK.md` +- Create or update: `benchmarks/swebench_lite/public-benchmark.json` +- Test: site and public-artifact validation tests. + +- [ ] Add a top-of-page benchmark strip showing correctness first, then verified token + delta, actions prevented, and governance overhead. +- [ ] Label smoke/canary sample size and exploratory status visibly. +- [ ] Link the exact workflow run, machine-readable artifact, methodology, task manifest, + source commit, and reproduction command. +- [ ] Render negative, neutral, or regressive outcomes without euphemism. +- [ ] Add responsive and reduced-motion behavior; keep the site dependency-free. +- [ ] Run local HTTP/browser checks for layout, links, console errors, and mobile viewport. + +## Task 12: Final verification and publication + +- [ ] Re-run all format, lint, type, unit, integration, schema, evidence, secret, and site + checks from the final tree. +- [ ] Confirm README/site numbers are derived from the committed JSON artifact and contain + no manually divergent totals. +- [ ] Commit intentionally, push the benchmark branch, and open a draft PR with exact test + and verifier evidence. +- [ ] Confirm GitHub Actions CI, SWE-bench workflow, and Pages deployment pass. +- [ ] Report the branch/PR, workflow run, site URL, exact measured outcome, limitations, and + reproduction path; only then mark the goal complete. diff --git a/docs/superpowers/specs/2026-08-11-codex-benchmark-integration-design.md b/docs/superpowers/specs/2026-08-11-codex-benchmark-integration-design.md new file mode 100644 index 0000000..072195a --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-codex-benchmark-integration-design.md @@ -0,0 +1,225 @@ +# Codex + MARGINAL Paired Benchmark Design + +## Status and amendment record + +This design is frozen before comparative inference. It supersedes the host-checkout +feasibility design written earlier on 2026-08-11. The feasibility review found two +scientific blockers: Codex 0.147 does not expose a shell exit code in `PostToolUse`, and a +host checkout does not reproduce the official SWE-bench task environment. + +The approved amendment is: + +1. run the whole Codex process inside the official per-instance SWE-bench image; +2. define an executed action as a tool invocation with an authoritative `PostToolUse`; +3. reserve `settle_failure` for an invocation that did not complete and therefore has no + valid post event; +4. verify patches with the official SWE-bench harness on Modal; +5. publish no efficiency or correctness claim until both lanes pass integrity checks and + the official verifier result has been merged. + +This amendment is methodological, not outcome-driven: both public result files are empty +at freeze time. + +## Objective + +Measure whether the released MARGINAL diminishing-return policy changes Codex behavior, +token use, latency, and task resolution under a matched OFF/ON intervention. Correctness +is primary. Token savings count only when paired correctness is preserved. Integration +latency and policy overhead count against MARGINAL. + +The implementation is pinned to MARGINAL commit +`4c8856401b4c752d5c214df5e84b9632d9897ec9`, Codex CLI `0.147.0`, model +`gpt-5.6-sol`, reasoning effort `high`, the frozen prompt hash, and a manifest of exact +SWE-bench instance IDs and base commits. + +## Considered architectures + +### A. Full container execution — selected + +Codex, its shell tools, the MARGINAL daemon, and hooks run inside the official task image. +The task checkout is mounted at `/testbed`. A separate tool layer provides pinned Codex +and an isolated MARGINAL Python environment without changing the task dependency +environment. This gives the strongest execution parity and makes every local tool action +observable at the same boundary. + +### B. Host Codex with a container shell bridge — rejected + +Codex 0.147 resolves the default user shell from the passwd database; setting `$SHELL` +does not reliably redirect every command. A prompt instruction would not be an enforceable +experimental boundary. + +### C. Cloud generation and verification — deferred + +Running inference and verification entirely in GitHub Actions or Modal is reproducible but +adds credential provisioning and cloud-executor differences that are unnecessary to test +the intervention. Modal remains the authoritative verifier. + +## Matched experimental boundary + +Both lanes use: + +- the same official task image digest and `/testbed` base commit; +- the same overlay image digest, Codex binary, model, effort, prompt, timeout, sandbox, + network policy, environment allowlist, and host architecture; +- an independent fresh container, checkout, Codex home, and run directory; +- the same external JSONL collector and patch extraction logic. + +The OFF lane has no MARGINAL process, hook, prompt text, configuration, state, or context. +The ON lane adds only the MARGINAL daemon and official `PreToolUse`/`PostToolUse` hooks. +Their CPU time, wall time, denials, errors, and token consequences are intervention cost. + +Hosted tools that bypass the official hook surface are disabled symmetrically. The causal +estimand is therefore **MARGINAL control over hook-covered local Codex tool actions in an +official SWE-bench environment**, not governance of model inference or every hosted tool. + +## Runtime architecture + +```text +host orchestrator + |-- materialize exact SWE-bench instance and official image digest + |-- create clean OFF or ON worktree and isolated run directory + `-- start one disposable task container with model API egress + |-- codex exec --ephemeral --json in /testbed + | |-- local tools execute inside the task container + | `-- ON only: PreToolUse/PostToolUse hooks + | `-- Unix socket --> MARGINAL daemon/Treasury + `-- bind-mounted raw events, trace, metrics, and final patch + +strict run records + predictions + --> evidence protocol validation + --> GitHub Actions SWE-bench workflow + --> official Modal verification + --> paired merge and public benchmark artifacts +``` + +The task container retains network access for the Codex parent process to reach the model +API. Codex `workspace-write` sandbox policy disables network for model-generated tool +subprocesses. Tool subprocesses receive only an allowlisted environment and never receive +model credentials. + +Authentication preference is an ephemeral `OPENAI_API_KEY` injected only into the Codex +parent. If the pinned model cannot use API-key authentication, the run stops; it must not +fall back to copying a broadly readable user home or credential file into the task image. +No credential is serialized into a command, image layer, trace, patch, or public artifact. + +## Action normalization + +Each `PreToolUse` payload becomes one MARGINAL `Action`: + +- `name`: canonical tool name plus a compact normalized operation label; +- `kind`: `shell`, `edit`, `mcp`, `verification`, or `tool`; +- `fingerprint`: the unique Codex tool invocation ID, settling a reservation once; +- `metadata.marginal_semantic_key`: SHA-256 of canonical tool name and normalized input; +- `metadata.state_hash`: observable workspace state immediately before execution; +- `metadata.evidence_hash`: last completed evidence for that semantic action; +- `metadata.phase`: stable `codex-tool-use` phase; +- `expected_gain`: omitted, selecting the shipped estimator default; +- `cost`: zero pre-execution tool cost because Codex hooks expose no defensible forecast of + future model tokens. + +The zero forecast cost means this experiment tests exact, state-aware repetition control, +not a synthetic economic ROI estimate. Actual tokens, latency, actions, and verifier result +are measured externally. + +The workspace hash covers tracked content, staged and unstaged changes, and untracked task +files while excluding `.git`, `.codex`, runtime output, caches, and virtual environments. +It stores hashes, never task source. + +## Completion, failure, and evidence semantics + +Codex 0.147 emits `PostToolUse` only after its tool handler completes but its unified shell +post payload does not expose the command exit status. A completed test that exits nonzero +is still a real evidence-producing action: it may reveal the defect and change the next +decision. Treating every red test as a transport failure would make repeated verification +invisible and eliminate the mechanism being measured. + +Therefore: + +- a well-formed matching `PostToolUse` commits the action and records the post state and + evidence hash, regardless of the command's application-level exit code; +- missing, malformed, duplicate, or identity-mismatched post events are integration + failures, not successful actions; +- a denied pre action is settled as denied and never enters successful repetition history; +- runner termination with pending actions fails the ON run; +- complete hook coverage is exact: committed + denied + explicitly failed decisions must + equal every hook-coverable Codex tool call, with one pre/post identity chain per executed + action. + +The frozen unproductive-repeat definition is: the same normalized invocation is proposed +after a completed execution, against unchanged observable workspace state, without new +evidence. The default detector discounts the second proposal and recommends stopping from +the third same-state proposal onward. + +## Integrity, privacy, and fail-closed gates + +Every public run must satisfy all of the following: + +- exact instance ID, base commit, task-set hash, prompt hash, Codex version, model, effort, + timeout, sandbox, image digest, overlay digest, source commit, and configuration hash; +- detached clean initial worktree and no run directory inside `/testbed`; +- strict Codex JSONL lifecycle and token accounting; +- exact ON hook coverage and no OFF MARGINAL footprint; +- no pending reservations or daemon errors at shutdown; +- a final patch extracted only from the task worktree; +- fail-closed secret scanning of raw public fields, patches, predictions, and generated + reports before acceptance; +- no prompt, task source, raw tool output, credential, private ledger key, or user-home + material in public artifacts. + +Any mismatch produces an explicit infrastructure or integration failure. It is never +silently converted into an unresolved benchmark row. + +## Benchmark stages and promotion gates + +1. Unit and protocol suite with deterministic fake event streams. +2. Container integration fixture proving API auth, shell execution, hook coverage, denial, + patch extraction, and secret scanning. +3. Frozen three-task paired smoke: six independent inference runs. +4. Official Modal verification and paired merge of the smoke. +5. Frozen twenty-task engineering canary: forty independent runs, only if the smoke has + complete telemetry and the verifier pipeline is green. +6. Official Modal verification and paired merge of the canary. +7. README/site publication only from committed machine-readable verifier artifacts. + +An integrity failure is fixed and the affected stage rerun from clean state. Policy +thresholds, task membership, prompt, timeout, and analysis rules are not tuned after lane +outcomes are observed. + +## Outcomes and analysis + +The official verifier alone owns `resolved`. Paired outcomes are `BOTH_SOLVE`, +`BASELINE_ONLY`, `MARGINAL_ONLY`, and `NEITHER`. Primary reporting always includes: + +- resolved count/rate by lane and paired outcome counts; +- total/input/cached/output/reasoning tokens by lane and paired differences; +- wall time and governance overhead; +- tool calls, denials, reroutes, repeated proposals, and hook coverage; +- infrastructure/integration failures as a separate denominator; +- per-task rows sufficient to recompute every aggregate. + +Token savings are reported as correctness-preserving only for `BOTH_SOLVE` pairs or as a +clearly labeled all-pairs engineering metric. If ON regresses correctness, the headline +must show the regression before any token reduction. Confidence intervals and statistical +tests are included when the sample supports them; a smoke or twenty-task canary remains +exploratory and is never described as publication-grade proof. + +Intervention labels are `BENEFICIAL`, `NEUTRAL`, `HARMFUL`, or `INDETERMINATE`. Without +explicit counterfactual or reviewer evidence, a trajectory-level outcome does not assign +causal value to an individual denial. + +## Publication contract + +The root README and website may show benchmark data at the top only after the official +artifact has been validated against its schema and provenance. The headline links to the +machine-readable artifact, exact workflow run, methodology, task manifest, source commit, +and limitations. If the result is negative or neutral, it is published unchanged. If only +the smoke completes, it is labeled an integration smoke and not promoted as a performance +claim. + +## Non-goals + +- Tuning MARGINAL policy or estimator on benchmark outcomes. +- Claiming causal value for individual actions from final task success. +- Treating synthetic demonstrations, the smoke, or the canary as broad model evidence. +- Hiding regressions, failed runs, intervention overhead, or negative results. +- Generalizing beyond the pinned model, version, tasks, policy, and official environment. diff --git a/pyproject.toml b/pyproject.toml index dfa79ce..3d44688 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ marginal = ["py.typed", "schemas/*.json"] [tool.pytest.ini_options] addopts = "-ra" -pythonpath = ["src"] +pythonpath = ["src", "."] testpaths = ["tests"] [tool.ruff] diff --git a/site/benchmark-data.json b/site/benchmark-data.json new file mode 100644 index 0000000..8c048dd --- /dev/null +++ b/site/benchmark-data.json @@ -0,0 +1 @@ +{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": 0.0}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": 0.0}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": 0.0}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "quality": {"false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": true, "preserved_within_one_pp": true, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "tasks": 3} diff --git a/site/index.html b/site/index.html index 39acd91..9bd8d98 100644 --- a/site/index.html +++ b/site/index.html @@ -28,6 +28,7 @@ MARGINAL