From 79e4d28a31e01c35e01f204a71dae5b68224df1b Mon Sep 17 00:00:00 2001 From: chandan25809-lyzr Date: Tue, 20 Oct 2026 22:08:01 +0000 Subject: [PATCH 01/82] gitagent v2: a Rust agent harness for terminal tasks A single static binary that runs one agent loop over a persistent shell: native tool calling with a text fallback, PTY sessions with end markers, output shaping with spill files, an environment snapshot, batched masking and compaction, time and loop guards, and an evidence-based stop check. Provider adapters for Anthropic, OpenAI, Google and OpenAI-compatible endpoints; per-family profiles in YAML; ATIF trajectories and counters written every turn. Also included: the Harbor shim and benchmark scripts for Terminal-Bench 2.1 (dev/held-out split, rounds, pricing, dashboard), the offline lesson learner with its leakage audit, the .claude/ project-file layers, the gitagent-sdk library API, a Landlock+seccomp sandbox, and the integrity audit script. --- .github/workflows/ci.yml | 41 + .github/workflows/release.yml | 51 + .gitignore | 9 + BLUEPRINT.md | 329 ++ Cargo.lock | 2057 +++++++++++ Cargo.toml | 47 + README.md | 161 + TASKS.md | 117 + bench/RESULTS.md | 32 + bench/fixtures/smoke.yaml | 12 + bench/leaderboard.json | 112 + bench/lessons/lessons.json | 3216 ++++++++++++++++++ bench/pricing.json | 23 + bench/profiles/anthropic-m3.yaml | 64 + bench/profiles/anthropic-m4-nolessons.yaml | 65 + bench/profiles/anthropic-m4.yaml | 64 + bench/profiles/openai-m4.yaml | 63 + bench/rounds.json | 219 ++ bench/split.json | 97 + crates/cli/Cargo.toml | 33 + crates/cli/src/learn_cmd.rs | 155 + crates/cli/src/main.rs | 467 +++ crates/cli/src/otel.rs | 331 ++ crates/cli/src/repl.rs | 66 + crates/cli/src/repo_mode.rs | 206 ++ crates/cli/src/sandbox_exec.rs | 50 + crates/cli/tests/cli.rs | 700 ++++ crates/context/Cargo.toml | 13 + crates/context/src/lib.rs | 178 + crates/context/src/tests.rs | 109 + crates/core/Cargo.toml | 26 + crates/core/src/fallback.rs | 141 + crates/core/src/lib.rs | 1286 +++++++ crates/core/src/snapshot.rs | 98 + crates/core/src/tests.rs | 1200 +++++++ crates/guards/Cargo.toml | 13 + crates/guards/src/lib.rs | 10 + crates/guards/src/loops.rs | 109 + crates/guards/src/stop.rs | 197 ++ crates/guards/src/time.rs | 124 + crates/hooks/Cargo.toml | 20 + crates/hooks/src/lib.rs | 685 ++++ crates/learn/Cargo.toml | 21 + crates/learn/src/audit.rs | 123 + crates/learn/src/distill.rs | 126 + crates/learn/src/lib.rs | 292 ++ crates/learn/src/store.rs | 301 ++ crates/learn/src/tests.rs | 252 ++ crates/learn/src/trace.rs | 164 + crates/loader/Cargo.toml | 18 + crates/loader/src/agents.rs | 218 ++ crates/loader/src/config.rs | 58 + crates/loader/src/lib.rs | 682 ++++ crates/loader/src/profile.rs | 235 ++ crates/loader/src/sdk.rs | 256 ++ crates/loader/src/skills.rs | 130 + crates/provider/Cargo.toml | 22 + crates/provider/src/anthropic.rs | 462 +++ crates/provider/src/anthropic/tests.rs | 428 +++ crates/provider/src/compat.rs | 306 ++ crates/provider/src/compat/ollama-stream.txt | 8 + crates/provider/src/compat/tests.rs | 135 + crates/provider/src/google.rs | 335 ++ crates/provider/src/google/tests.rs | 181 + crates/provider/src/lib.rs | 83 + crates/provider/src/openai.rs | 412 +++ crates/provider/src/openai/tests.rs | 352 ++ crates/provider/src/scripted.rs | 176 + crates/provider/src/sse.rs | 44 + crates/provider/src/test_server.rs | 52 + crates/provider/src/types.rs | 200 ++ crates/record/Cargo.toml | 17 + crates/record/src/atif.rs | 352 ++ crates/record/src/counters.rs | 92 + crates/record/src/events.rs | 116 + crates/record/src/lib.rs | 30 + crates/record/src/time.rs | 56 + crates/repo/Cargo.toml | 15 + crates/repo/src/lib.rs | 290 ++ crates/repo/src/tests.rs | 223 ++ crates/sandbox/Cargo.toml | 20 + crates/sandbox/src/lib.rs | 192 ++ crates/sdk/Cargo.toml | 25 + crates/sdk/src/lib.rs | 999 ++++++ crates/sdk/src/tests.rs | 349 ++ crates/shaper/Cargo.toml | 14 + crates/shaper/src/lib.rs | 322 ++ crates/shaper/src/tests.rs | 139 + crates/terminal/Cargo.toml | 17 + crates/terminal/src/buffer.rs | 202 ++ crates/terminal/src/keys.rs | 70 + crates/terminal/src/lib.rs | 524 +++ crates/terminal/src/manager.rs | 254 ++ crates/terminal/src/tests.rs | 146 + crates/tools/Cargo.toml | 24 + crates/tools/src/exec.rs | 178 + crates/tools/src/image_read.rs | 125 + crates/tools/src/lib.rs | 704 ++++ crates/tools/src/naming.rs | 151 + crates/tools/src/patch.rs | 562 +++ crates/tools/src/read_file.rs | 112 + crates/tools/src/skill.rs | 30 + crates/tools/src/stdin.rs | 35 + crates/tools/src/str_replace.rs | 174 + crates/tools/src/todo.rs | 69 + crates/tools/src/write_file.rs | 77 + docs/DEVIATIONS.md | 185 + docs/INTEGRITY.md | 36 + docs/LEARNING.md | 110 + gitagent.yaml | 15 + gitagent_harbor/__init__.py | 0 gitagent_harbor/agent.py | 230 ++ profiles/anthropic.yaml | 62 + profiles/compat.yaml | 58 + profiles/google.yaml | 58 + profiles/openai.yaml | 61 + prompts/learner.md | 22 + prompts/lessons.md | 17 + prompts/system.md | 37 + rust-toolchain.toml | 4 + scripts/build-release.sh | 21 + scripts/dashboard.py | 2491 ++++++++++++++ scripts/install.sh | 56 + scripts/integrity-audit.sh | 62 + scripts/make-split.py | 127 + scripts/package-release.sh | 38 + scripts/run-dev-pass.sh | 65 + 127 files changed, 29528 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 BLUEPRINT.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 TASKS.md create mode 100644 bench/RESULTS.md create mode 100644 bench/fixtures/smoke.yaml create mode 100644 bench/leaderboard.json create mode 100644 bench/lessons/lessons.json create mode 100644 bench/pricing.json create mode 100644 bench/profiles/anthropic-m3.yaml create mode 100644 bench/profiles/anthropic-m4-nolessons.yaml create mode 100644 bench/profiles/anthropic-m4.yaml create mode 100644 bench/profiles/openai-m4.yaml create mode 100644 bench/rounds.json create mode 100644 bench/split.json create mode 100644 crates/cli/Cargo.toml create mode 100644 crates/cli/src/learn_cmd.rs create mode 100644 crates/cli/src/main.rs create mode 100644 crates/cli/src/otel.rs create mode 100644 crates/cli/src/repl.rs create mode 100644 crates/cli/src/repo_mode.rs create mode 100644 crates/cli/src/sandbox_exec.rs create mode 100644 crates/cli/tests/cli.rs create mode 100644 crates/context/Cargo.toml create mode 100644 crates/context/src/lib.rs create mode 100644 crates/context/src/tests.rs create mode 100644 crates/core/Cargo.toml create mode 100644 crates/core/src/fallback.rs create mode 100644 crates/core/src/lib.rs create mode 100644 crates/core/src/snapshot.rs create mode 100644 crates/core/src/tests.rs create mode 100644 crates/guards/Cargo.toml create mode 100644 crates/guards/src/lib.rs create mode 100644 crates/guards/src/loops.rs create mode 100644 crates/guards/src/stop.rs create mode 100644 crates/guards/src/time.rs create mode 100644 crates/hooks/Cargo.toml create mode 100644 crates/hooks/src/lib.rs create mode 100644 crates/learn/Cargo.toml create mode 100644 crates/learn/src/audit.rs create mode 100644 crates/learn/src/distill.rs create mode 100644 crates/learn/src/lib.rs create mode 100644 crates/learn/src/store.rs create mode 100644 crates/learn/src/tests.rs create mode 100644 crates/learn/src/trace.rs create mode 100644 crates/loader/Cargo.toml create mode 100644 crates/loader/src/agents.rs create mode 100644 crates/loader/src/config.rs create mode 100644 crates/loader/src/lib.rs create mode 100644 crates/loader/src/profile.rs create mode 100644 crates/loader/src/sdk.rs create mode 100644 crates/loader/src/skills.rs create mode 100644 crates/provider/Cargo.toml create mode 100644 crates/provider/src/anthropic.rs create mode 100644 crates/provider/src/anthropic/tests.rs create mode 100644 crates/provider/src/compat.rs create mode 100644 crates/provider/src/compat/ollama-stream.txt create mode 100644 crates/provider/src/compat/tests.rs create mode 100644 crates/provider/src/google.rs create mode 100644 crates/provider/src/google/tests.rs create mode 100644 crates/provider/src/lib.rs create mode 100644 crates/provider/src/openai.rs create mode 100644 crates/provider/src/openai/tests.rs create mode 100644 crates/provider/src/scripted.rs create mode 100644 crates/provider/src/sse.rs create mode 100644 crates/provider/src/test_server.rs create mode 100644 crates/provider/src/types.rs create mode 100644 crates/record/Cargo.toml create mode 100644 crates/record/src/atif.rs create mode 100644 crates/record/src/counters.rs create mode 100644 crates/record/src/events.rs create mode 100644 crates/record/src/lib.rs create mode 100644 crates/record/src/time.rs create mode 100644 crates/repo/Cargo.toml create mode 100644 crates/repo/src/lib.rs create mode 100644 crates/repo/src/tests.rs create mode 100644 crates/sandbox/Cargo.toml create mode 100644 crates/sandbox/src/lib.rs create mode 100644 crates/sdk/Cargo.toml create mode 100644 crates/sdk/src/lib.rs create mode 100644 crates/sdk/src/tests.rs create mode 100644 crates/shaper/Cargo.toml create mode 100644 crates/shaper/src/lib.rs create mode 100644 crates/shaper/src/tests.rs create mode 100644 crates/terminal/Cargo.toml create mode 100644 crates/terminal/src/buffer.rs create mode 100644 crates/terminal/src/keys.rs create mode 100644 crates/terminal/src/lib.rs create mode 100644 crates/terminal/src/manager.rs create mode 100644 crates/terminal/src/tests.rs create mode 100644 crates/tools/Cargo.toml create mode 100644 crates/tools/src/exec.rs create mode 100644 crates/tools/src/image_read.rs create mode 100644 crates/tools/src/lib.rs create mode 100644 crates/tools/src/naming.rs create mode 100644 crates/tools/src/patch.rs create mode 100644 crates/tools/src/read_file.rs create mode 100644 crates/tools/src/skill.rs create mode 100644 crates/tools/src/stdin.rs create mode 100644 crates/tools/src/str_replace.rs create mode 100644 crates/tools/src/todo.rs create mode 100644 crates/tools/src/write_file.rs create mode 100644 docs/DEVIATIONS.md create mode 100644 docs/INTEGRITY.md create mode 100644 docs/LEARNING.md create mode 100644 gitagent.yaml create mode 100644 gitagent_harbor/__init__.py create mode 100644 gitagent_harbor/agent.py create mode 100644 profiles/anthropic.yaml create mode 100644 profiles/compat.yaml create mode 100644 profiles/google.yaml create mode 100644 profiles/openai.yaml create mode 100644 prompts/learner.md create mode 100644 prompts/lessons.md create mode 100644 prompts/system.md create mode 100644 rust-toolchain.toml create mode 100755 scripts/build-release.sh create mode 100644 scripts/dashboard.py create mode 100755 scripts/install.sh create mode 100755 scripts/integrity-audit.sh create mode 100644 scripts/make-split.py create mode 100755 scripts/package-release.sh create mode 100755 scripts/run-dev-pass.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..3573859c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: ci + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install toolchain from rust-toolchain.toml + run: rustup show active-toolchain || rustup toolchain install + - uses: Swatinem/rust-cache@v2 + - run: cargo fmt --all -- --check + - run: cargo clippy --all-targets --locked -- -D warnings + - run: cargo test --workspace --locked + + musl: + runs-on: ubuntu-latest + needs: check + steps: + - uses: actions/checkout@v4 + - name: Install toolchain from rust-toolchain.toml + run: rustup show active-toolchain || rustup toolchain install + - uses: Swatinem/rust-cache@v2 + - uses: mlugg/setup-zig@v2 + - run: cargo install --locked cargo-zigbuild + - run: scripts/build-release.sh + - name: Check the binaries are static + run: | + for t in x86_64-unknown-linux-musl aarch64-unknown-linux-musl; do + file target/$t/release/gitagent | tee /dev/stderr | grep -q 'statically linked' + done + - uses: actions/upload-artifact@v4 + with: + name: gitagent-linux + path: target/*-unknown-linux-musl/release/gitagent diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..422531f3 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: release + +on: + push: + tags: ["v*"] + +permissions: + contents: write + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install toolchain from rust-toolchain.toml + run: rustup show active-toolchain || rustup toolchain install + - uses: mlugg/setup-zig@v2 + - run: cargo install --locked cargo-zigbuild + - run: scripts/package-release.sh x86_64-unknown-linux-musl aarch64-unknown-linux-musl + - uses: actions/upload-artifact@v4 + with: + name: linux + path: dist/*.tar.gz + + macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Install toolchain from rust-toolchain.toml + run: rustup show active-toolchain || rustup toolchain install + - run: scripts/package-release.sh aarch64-apple-darwin x86_64-apple-darwin + - uses: actions/upload-artifact@v4 + with: + name: macos + path: dist/*.tar.gz + + publish: + needs: [linux, macos] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: Checksums + working-directory: dist + run: sha256sum gitagent-*.tar.gz > SHA256SUMS + - name: Create the release + env: + GH_TOKEN: ${{ github.token }} + run: gh release create "$GITHUB_REF_NAME" dist/* --repo "$GITHUB_REPOSITORY" --title "$GITHUB_REF_NAME" --generate-notes diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..05da2bf7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/target/ +bench/jobs/ +.env +*.env +.DS_Store +.env.* +__pycache__/ +*.pyc +dist/ diff --git a/BLUEPRINT.md b/BLUEPRINT.md new file mode 100644 index 00000000..bf760769 --- /dev/null +++ b/BLUEPRINT.md @@ -0,0 +1,329 @@ +# gitagent v2 — Rust Engine Blueprint + +`BLUEPRINT.md` says **what** to build and **why**; `TASKS.md` says **in what order** and **how to know each step is done**. + +--- + +## 0. Goal in one paragraph + +Build gitagent v2 **from scratch** as a single static **Rust** binary. It keeps the gitagent name and one idea from v1 (all agent behavior lives in version-controlled files) but reuses no v1 code and keeps no v1 compatibility. Its core is an **evidence-gated minimal loop** proven to score well on Terminal-Bench 2.1: one agent, one append-only history, about seven tools, persistent PTY sessions, an environment snapshot, output shaping with spill files, batched observation masking, time and loop guards, and a stop check that refuses "done" until the agent has verified its work. Target: on the same model, beat Terminus 2 by 5+ points on a frozen held-out split, with median time at or below Terminus 2, timeouts under 10%, and ≥80% of input tokens served from the provider cache. + + +--- + +## 1. Requirements (every design choice traces to one) + +| ID | Requirement | +|---|---| +| R1 | Higher Terminal-Bench 2.1 pass rate than Terminus 2 on the same model (paired, held-out, ≥3 trials) | +| R2 | Works with OpenAI, Anthropic, Google and OpenAI-compatible open-weight endpoints | +| R3 | Median agent time ≤ Terminus 2; timeouts < 10% | +| R4 | ≥80% of input tokens from provider cache on long tasks | +| R5 | Handles REPLs, servers, long builds, Ctrl-C | +| R6 | Never accepts "done" without verification after the last edit | +| R7 | Runs in any Linux container with no dependencies (static musl binary) | +| R8 | All tunable behavior in config (YAML), not code | +| R9 | Every feature emits fired/consumed counters | +| R10 | Trajectories exported in Harbor's ATIF format, written every turn | +| R11 | Integrity: no internet solution lookup, no task-name routing, no writes to `/tests`, no reading verifier code | +| R12 | All behavior lives in version-controlled files (config, prompts, profiles), so every score traces to a commit | + +--- + +## 2. Architecture + +``` + ┌───────────────────────────────┐ + Harbor / CLI / TS SDK │ gitagent (static Rust binary) │ Model APIs + ─────────────────────▶ │ │ ◀────▶ OpenAI, Anthropic, + instruction + flags │ loader ── config + profiles │ Google, open-weight + │ snapshot ─ env facts turn 0 │ + │ controller ── THE LOOP │ + │ ├ provider (native tools) │ + │ ├ tools: exec/stdin/read/ │ + │ │ edit/image/todo/ │ + │ │ complete + yaml │ + │ ├ terminal (PTY sessions) │ + │ ├ shaper (cap + spill) │ + │ ├ guards (time, loops) │ + │ ├ context (mask, compact) │ + │ ├ stop check (evidence) │ + │ └ hooks (optional gates) │ + │ record ── ATIF + counters │ + │ sandbox ── Landlock/seccomp │ + └───────────────────────────────┘ +``` + +### Run lifecycle (numbers are referenced throughout) + +1. **Start** — CLI/Harbor/SDK starts the binary with an instruction. +2. **Load** — read `gitagent.yaml` + model profile; build the system prompt from `prompts/system.md` (+ optional `prompts/rules.md` and the skill index). System prompt must be byte-identical across tasks (cache). +3. **Snapshot** — one compound probe (≤15 s, fail-silent) → injected into the **first user message**, never the system prompt. +4. **Model call** — provider adapter, native tools, effort from profile; text fallback + one-line repair request on malformed calls. +5. **Dispatch** — run the tool; shell commands go to PTY sessions with end markers; `pre_tool_use` hooks run first. +6. **Shape** — strip ANSI, keep last progress-bar frame, collapse similar lines, cap (~8K tokens, 30% head / 70% tail), save full output to a spill file, add footer. +7. **Guards** — remaining time in footer; nudge at 70%, force stop check at 90%; 3 identical (command, normalized output) → change-approach message. +8. **Context** — every ~50K tokens of growth, mask tool results older than the last 8 into stubs; structured compaction only near the limit. +9. **Stop check** — `task_complete` accepted only if a verification ran after the last edit; else reject (max 2), re-show instruction. +10. **Record** — append trajectory step + counters to disk **every turn**; final flush on exit. +11. **Grade** (benchmark only) — Harbor runs hidden tests. + +Steps 4–9 repeat until 9 accepts or a limit is hit. + +--- + +## 3. Design decisions (short form) + +| # | Decision | Why | +|---|---|---| +| D1 | Static Rust binary inside the container | No deps, ms startup, direct PTY, memory-safe, sandbox-ready | +| D2 | Single agent, linear append-only history | Single loops are where the measurable gains are; caching needs it | +| D3 | Native tool calling + text fallback + repair | Reliable calls on every provider, open-weight models included | +| D4 | PTY sessions, end markers, background jobs | No guessed waits; REPLs/servers work | +| D5 | ~7 tools | Small cached prefix; fewer bad choices | +| D6 | Edit tool per model family | apply_patch for GPT, str_replace for Claude | +| D7 | Environment snapshot in first user message | Saves 2–4 turns; cache-safe placement | +| D8 | Shaper + spill files | Biggest token cost; nothing lost | +| D9 | Batched observation masking + cache breakpoints | ~½ cost at same solve rate; keeps cache | +| D10 | Structured compaction near limit only | Keeps requirements and failed approaches | +| D11 | Evidence-based stop check, no answer rewriting | Premature "done" is a top failure; rewriting critics hurt | +| D12 | Time budget + loop breaker | Timeouts are failures | +| D13 | Per-model-family profiles in YAML | Profiles don't transfer across providers | +| D14 | Own thin provider adapters (reqwest+serde) | Byte control for cache hits | +| D15 | Counters + ATIF written every turn | Dead features show up; timeouts keep their record | +| D16 | Landlock + seccomp sandbox, profile-switchable | Safe outside benchmarks | + +**Excluded on purpose:** critics that rewrite answers, semantic tool-result caches, cross-task memory during benchmarks, multi-agent by default, best-of-N on Terminal-Bench, automated harness rewriting, masking every turn, third-party multi-provider frameworks. + +--- + +## 4. Product surface (v2, from scratch) + +No v1 code, flags, file formats or SDK types are carried over. The product grows in this order: + +| Capability | v2 design | Milestone | +|---|---|---| +| Benchmark mode | `gitagent --bench --instruction-file … --model … --profile …`; no network except the model API, no memory, no git commits, `/tests` writes denied | M1 | +| Single prompt | `gitagent -p "…" --dir `: runs the same loop on a local folder, prints a summary | M6 | +| Interactive | `gitagent --dir `: REPL on top of the same loop; Ctrl-C interrupts the current turn | M6 | +| Config as files | `gitagent.yaml`, `prompts/`, `profiles/`, optional `skills/` and `hooks/` in the project or `~/.gitagent/` | M6 | +| Repo mode | `gitagent --repo -p "…"`: clone the repo, create a new working branch, run the loop inside it, commit, and optionally push (§5 Repo mode) | M6 | +| Hooks (optional) | `pre_tool_use` / `post_tool_use` scripts get JSON on stdin and return `allow`, `block` or `modify` | M6 | +| Skills (optional) | `skills//SKILL.md`; index in the system prompt, full text loaded on demand by a `skill` tool | M6 | +| Sandbox, telemetry | Landlock/seccomp on outside `--bench`; OpenTelemetry off unless configured | M7 | +| Later, only if needed | Opening a pull request, MCP client, a TypeScript or Python SDK over `--events jsonl` | after M7 | + +--- + +## 5. Crates and interfaces + +Cargo workspace, edition 2021, `#![forbid(unsafe_code)]` everywhere except `sandbox`. + +``` +gitagent/ +├─ crates/ +│ ├─ cli/ binary: clap flags, modes (interactive, -p, --bench, --repo) +│ ├─ loader/ gitagent.yaml + prompts + profiles → AgentSpec +│ ├─ provider/ Provider trait; anthropic, openai, google, compat +│ ├─ terminal/ PTY session manager, markers, background sessions +│ ├─ tools/ tool registry, built-ins, yaml tools, edit tools +│ ├─ shaper/ output shaping + spill +│ ├─ context/ history, masking, compaction, token estimates +│ ├─ guards/ time budget, loop detector, stop check +│ ├─ hooks/ optional allow/block/modify script hooks +│ ├─ record/ ATIF writer, counters, audit log, events (JSONL) +│ ├─ repo/ clone, branch, commit, push via the git CLI +│ ├─ sandbox/ landlock + seccomp (linux), no-op elsewhere +│ └─ core/ controller loop wiring everything +├─ profiles/ anthropic.yaml, openai.yaml, google.yaml, compat.yaml +├─ prompts/ system.md, rules.md +├─ harbor/ agent.py (Harbor installed-agent shim) +└─ tests/ fake-provider integration tests, fixtures +``` + +### Key types (sketch — refine, don't treat as final) + +```rust +// provider +pub trait Provider: Send + Sync { + async fn complete(&self, req: &ChatRequest) -> Result; + fn family(&self) -> Family; // Anthropic | OpenAI | Google | Compat +} +pub struct ChatRequest { pub system: String, pub messages: Vec, pub tools: Vec, + pub effort: Effort, pub temperature: Option, pub cache: CachePlan } +pub struct ChatResponse { pub text: String, pub tool_calls: Vec, pub usage: Usage /* incl. cache_read/write */ } + +// terminal +pub struct ExecResult { pub session: SessionId, pub output: Vec, pub exit: Option, pub finished: bool } +pub trait Terminal { + async fn exec(&mut self, cmd: &str, yield_ms: u64, background: bool) -> Result; + async fn write_stdin(&mut self, s: SessionId, keys: &str) -> Result; + async fn poll(&mut self, s: SessionId, yield_ms: u64) -> Result; + async fn kill(&mut self, s: SessionId) -> Result<()>; +} + +// shaper +pub struct Shaped { pub text: String, pub spill_path: PathBuf, pub total_lines: usize, pub hidden_lines: usize } +pub fn shape(raw: &[u8], cfg: &ShapeCfg, spill_dir: &Path, seq: u64) -> Shaped; + +// guards +pub enum StopVerdict { Accept, Reject { reason: String } } +pub struct StopCheck { last_edit_turn: Option, last_verify_turn: Option, rejections: u8 } +``` + +### Algorithms + +**Exec with end marker (terminal)** +``` +seq += 1; marker = "__GA_END_{seq}__" +write: "{cmd}\nprintf '\\n{marker} %s\\n' $?\n" +read PTY until line starting with marker OR yield_ms elapsed +if marker seen → finished=true, exit=parsed code, strip marker line +else → finished=false, keep session alive, return partial output + session id +``` + +**Shaper** +``` +write raw bytes to {spill_dir}/{seq}.log (always, first) +if binary (>10% non-text bytes) → "[binary output, N bytes, saved to {path}]" +text = strip_ansi(raw); for each line keep text after last '\r' +collapse runs ≥3 of lines equal after digits→'#' normalization → first line + "[N similar lines hidden]" +if tokens(text) > cap: keep head 30% + "[… N lines hidden — see {path} …]" + tail 70% +footer: "exit {code} · {total} lines · full log: {path} · {mins} min left" +``` + +**Batched masking (context)** +``` +after each turn: if est_tokens(history) - tokens_at_last_checkpoint ≥ mask_batch_tokens: + for tool results older than the last keep_last: replace body with stub + "[output hidden: exit {code}, {lines} lines → {spill_path}]" + tokens_at_last_checkpoint = est_tokens(history) +never mutate history between checkpoints (cache prefix must stay stable) +if remaining_context < compact_at_tokens: replace history with structured summary: + REQUIREMENTS (original instruction verbatim) | DONE | PENDING | KEY FILES | LAST FAILURE | RULED OUT +``` + +**Stop check (guards)** +``` +on edit tool call or exec matching edit_patterns (sed -i, >, >>, tee, mv, cp, patch, git apply, …): last_edit_turn = turn +on exec matching verify_patterns (test runners, curl/wget, running a built binary, python -c, node -e, + git clone/push, psql -c, …) with finished=true: last_verify_turn = turn +on task_complete: + if last_edit_turn.is_some() && last_verify_turn < last_edit_turn && rejections < max_rejections: + rejections += 1; return Reject("No verification since your last change. Original task: …") + return Accept +patterns live in the profile (R8) +``` + +**Loop detector (guards)** +``` +key = hash(tool name + normalized args + normalized first/last 20 output lines) +if last 3 keys identical → append "You have repeated the same action with the same result. Change approach." + and raise effort one level for the next call if the provider supports it +``` + +**Repo mode (repo crate)** +``` +inputs: --repo , optional --base (default: remote default branch), + optional --branch (resume), token from GITAGENT_GIT_TOKEN or GITHUB_TOKEN +clone: git clone --filter=blob:none (token supplied via GIT_ASKPASS helper, + never embedded in the URL, never written to .git/config, never logged) +branch: if --branch given and exists on remote → git checkout (resume) + else → git checkout -b gitagent/-<6-char-id> origin/ +run: the normal loop with --dir ; the snapshot includes the branch name and git status +commit: after the stop check accepts (and optionally every N turns, profile setting): + git add -A && git commit -m "gitagent: " (author from config; skip if nothing changed) +push: only with --push: git push -u origin +rules: never commit to or push the base branch; never force-push; never rewrite history; + refuse to start if the working branch equals the base branch +output: final event lists branch, commit SHAs, and push status +``` + +**Snapshot** +``` +one sh -c with `timeout 15`: pwd; ls -la | head -40; uname -a; +for t in python3 pip node npm go rustc cargo gcc g++ java make cmake git docker psql: version or "absent"; +free -m; nproc; git status --short | head -20 (if repo); ls of README*, Makefile, package.json, pyproject.toml +→ "[Environment Snapshot]\n…" ; on any failure → omit silently, count `snapshot_failed` +``` + +--- + +## 6. Configuration + +### `gitagent.yaml` + +```yaml +name: gitagent +model: anthropic: # provider:model +fallback_models: [] # optional +max_turns: 200 +timeout_s: 1800 # in --bench, Harbor's limit wins +profile: auto # auto = pick by model family; or profiles/.yaml +sandbox: auto # auto = off in --bench (container isolates), on otherwise +spill_dir: /tmp/gitagent/out +skills: true +hooks: true +repo: # used only with --repo + branch_prefix: gitagent/ + commit_every_turns: 0 # 0 = commit only when the stop check accepts + author: "gitagent " + push: false # --push on the CLI overrides +``` + +### Profile (`profiles/anthropic.yaml`) + +```yaml +family: anthropic +tool_calling: native # native | native_with_text_fallback | text +edit_tool: str_replace # str_replace | apply_patch +effort: high +temperature: 1.0 # required by some providers when reasoning is on +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +context: { mask_batch_tokens: 50000, keep_last: 8, compact_at_tokens: 16000 } +guards: { nudge_at: 0.7, force_stop_at: 0.9, loop_repeats: 3 } +stop: { max_rejections: 2, verify_patterns: [...], edit_patterns: [...] } +cache: { breakpoints: [system, last_two_messages] } +``` +`openai.yaml` uses `edit_tool: apply_patch`; `compat.yaml` uses `native_with_text_fallback` and a lower `cap_tokens`. Tune values **only on the dev split**. + +--- + +## 7. Interfaces to the outside + +- **CLI**: `--bench`, `--instruction-file`, `-p/--prompt`, `--dir`, `--model provider:model`, `--profile`, `--events jsonl`, `--config`; repo mode: `--repo `, `--base `, `--branch ` (resume), `--push`. Token from `GITAGENT_GIT_TOKEN` or `GITHUB_TOKEN`. +- **Events (`--events jsonl`)**: one JSON object per line on stdout: `turn_start`, `model_text`, `tool_call`, `tool_result`, `stop_check`, `guard`, `turn_end`, `repo` (clone, branch, commit, push), `run_end` (with usage, counters, branch and commit SHAs). Designed for logs and a future SDK. +- **Harbor shim (`harbor/agent.py`)**: installed agent; uploads binary to `/opt/gitagent/gitagent`, runs `gitagent --bench --instruction-file … --model … --profile …`, copies `/tmp/gitagent/trajectory.atif.json`, `out/` and counters back. Never writes to `/tests`. + +--- + +## 8. Observability + +Counters (written every turn to `/tmp/gitagent/counters.json`, plus OTel when enabled): +`turns, tool_calls{name}, exec_timeouts, background_sessions, shaped_bytes_in, shaped_bytes_out, spill_reads, collapse_events, masks_applied, compactions, cache_read_tokens, cache_write_tokens, input_tokens, output_tokens, stop_rejections, stop_accepts, loop_triggers, nudges, snapshot_ok, snapshot_failed, malformed_calls, repairs, hook_blocks`. + +A feature whose counters stay at zero across a dev run is either broken or useless — investigate before keeping it. + +--- + +## 9. Testing + +- **Unit**: shaper (ANSI, `\r`, collapse, cap, binary), marker parsing, stop-check rules, loop hash, masking stubs, profile loading. +- **Integration with a fake provider**: scripted model responses (YAML fixtures) drive the real loop against a real PTY in a temp dir. No network in CI. +- **Prompt stability**: snapshot test that the built system prompt is byte-identical across two different tasks. +- **Benchmark**: Harbor on the dev split, ≥3 trials, paired per-task comparison vs the previous milestone. Held-out only at M5 and release. + +--- + +## 10. Suggested crates + +`tokio`, `portable-pty`, `reqwest` (rustls), `serde`, `serde_json`, a maintained YAML crate (check status — `serde_yaml` is archived), `clap`, `strip-ansi-escapes`, `regex`, `tiktoken-rs` (estimates), `sha2`, `uuid`, `tracing` + `tracing-subscriber`, `anyhow` + `thiserror`, `landlock`, `seccompiler`, `opentelemetry` + `opentelemetry-otlp` (M7); `rmcp` only if MCP is added later. Static build targets: `x86_64-unknown-linux-musl`, `aarch64-unknown-linux-musl`. + +--- + +## 11. Open questions for the owner + +1. First two model families to optimize for? +2. After M7, which comes first: opening pull requests, MCP, or an SDK? +3. Public release plan: open source from day one, or after the first held-out result? diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..6cdcd056 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2057 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "annotate-snippets" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" +dependencies = [ + "anstyle", + "memchr", + "unicode-width", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "gitagent-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "gitagent-core", + "gitagent-learn", + "gitagent-loader", + "gitagent-provider", + "gitagent-record", + "gitagent-repo", + "gitagent-sandbox", + "gitagent-sdk", + "gitagent-tools", + "reqwest", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "gitagent-context" +version = "0.1.0" +dependencies = [ + "gitagent-provider", + "serde_json", +] + +[[package]] +name = "gitagent-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "gitagent-context", + "gitagent-guards", + "gitagent-hooks", + "gitagent-loader", + "gitagent-provider", + "gitagent-record", + "gitagent-shaper", + "gitagent-terminal", + "gitagent-tools", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "gitagent-guards" +version = "0.1.0" +dependencies = [ + "anyhow", + "regex", +] + +[[package]] +name = "gitagent-hooks" +version = "0.1.0" +dependencies = [ + "async-trait", + "gitagent-loader", + "regex", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "gitagent-learn" +version = "0.1.0" +dependencies = [ + "anyhow", + "gitagent-loader", + "gitagent-provider", + "gitagent-record", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "gitagent-loader" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde-saphyr", + "serde_json", + "tempfile", +] + +[[package]] +name = "gitagent-provider" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "gitagent-loader", + "reqwest", + "serde", + "serde-saphyr", + "serde_json", + "thiserror 2.0.21", + "tokio", +] + +[[package]] +name = "gitagent-record" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "gitagent-repo" +version = "0.1.0" +dependencies = [ + "anyhow", + "tempfile", +] + +[[package]] +name = "gitagent-sandbox" +version = "0.1.0" +dependencies = [ + "anyhow", + "landlock", + "libc", + "seccompiler", + "tempfile", +] + +[[package]] +name = "gitagent-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "gitagent-core", + "gitagent-hooks", + "gitagent-loader", + "gitagent-provider", + "gitagent-record", + "gitagent-tools", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "gitagent-shaper" +version = "0.1.0" +dependencies = [ + "tempfile", +] + +[[package]] +name = "gitagent-terminal" +version = "0.1.0" +dependencies = [ + "anyhow", + "portable-pty", + "tempfile", + "tokio", +] + +[[package]] +name = "gitagent-tools" +version = "0.1.0" +dependencies = [ + "anyhow", + "gitagent-context", + "gitagent-loader", + "gitagent-provider", + "gitagent-record", + "gitagent-shaper", + "gitagent-terminal", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "granit-parser" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f99e46474f56bd905c56e817ebddcf377a611f94c53ac4649e4d3fa3c0cd0" +dependencies = [ + "arraydeque", + "smallvec", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "landlock" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cca98e95f35b29d469dade6724c6f96cec9236640f745a0e99b0334ec320ab1" +dependencies = [ + "enumflags2", + "libc", + "thiserror 2.0.21", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-pty" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a596a2b3d2752d94f51fac2d4a96737b8705dddd311a32b9af47211f08671e" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "downcast-rs", + "filedescriptor", + "lazy_static", + "libc", + "log", + "nix", + "serial2", + "shared_library", + "shell-words", + "winapi", + "winreg", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases 0.2.2", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases 0.2.2", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-saphyr" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8050abb251097357e24aff63ba2c52a6309ecb7d23a5474023df960a02694d8" +dependencies = [ + "annotate-snippets", + "encoding_rs_io", + "granit-parser", + "num-traits", + "serde_core", + "smallvec", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serial2" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16809bc35793b19ce4e0c53924bc0dce3937f15487997cfdaed936004180730" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "shared_library" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11" +dependencies = [ + "lazy_static", + "libc", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..c4ce945b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,47 @@ +[workspace] +resolver = "2" +members = ["crates/*"] + +[workspace.package] +version = "0.1.0" +edition = "2021" +rust-version = "1.89" +publish = false + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.lints.clippy] +dbg_macro = "warn" +todo = "warn" + +[workspace.dependencies] +gitagent-cli = { path = "crates/cli" } +gitagent-context = { path = "crates/context" } +gitagent-core = { path = "crates/core" } +gitagent-guards = { path = "crates/guards" } +gitagent-hooks = { path = "crates/hooks" } +gitagent-learn = { path = "crates/learn" } +gitagent-loader = { path = "crates/loader" } +gitagent-provider = { path = "crates/provider" } +gitagent-record = { path = "crates/record" } +gitagent-repo = { path = "crates/repo" } +gitagent-sandbox = { path = "crates/sandbox" } +gitagent-sdk = { path = "crates/sdk" } +gitagent-shaper = { path = "crates/shaper" } +gitagent-terminal = { path = "crates/terminal" } +gitagent-tools = { path = "crates/tools" } + +anyhow = "1.0.104" +async-trait = "0.1.92" +clap = { version = "4.6.7", features = ["derive", "env"] } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +thiserror = "2.0.21" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "process", "fs", "io-util", "io-std", "signal"] } + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = true +panic = "abort" diff --git a/README.md b/README.md new file mode 100644 index 00000000..aeb7e484 --- /dev/null +++ b/README.md @@ -0,0 +1,161 @@ +# gitagent + +A terminal agent in a single static binary. Its behavior lives in version-controlled files: `gitagent.yaml`, prompts, model profiles, skills and hooks. Any result can be traced to a commit. + +It works the same way in every mode. One loop calls the model, runs its tool calls in a persistent shell, and records every turn. It runs one task in a folder, an interactive session, a task in a fresh clone of a repository, or a Terminal-Bench task through Harbor. + +## Install + +```sh +curl -fsSL https://raw.githubusercontent.com/open-gitagent/gitagent/v2/scripts/install.sh | sh +``` + +This installs the binary for your system (Linux x86_64/arm64, macOS arm64/x86_64) into `~/.local/bin`; set `INSTALL_DIR` to change that. To build from source you need Rust (the version is pinned in `rust-toolchain.toml`): + +```sh +cargo build --release # ./target/release/gitagent +scripts/build-release.sh # static Linux binaries (needs cargo-zigbuild on macOS) +``` + +## Use + +Set the key for your model provider, then: + +```sh +export ANTHROPIC_API_KEY=... + +# One task in a folder; prints the agent's summary when done +gitagent -p "fix the failing test" --dir ./project --model anthropic:claude-opus-5-5 + +# Interactive: type tasks one per line; each continues the same session +gitagent --dir ./project --model anthropic:claude-opus-5-5 + +# Work in a fresh clone on a new branch, commit when done, push +GITHUB_TOKEN=... gitagent --repo https://github.com/org/app.git -p "fix the failing test" --push +``` + +In interactive mode, Ctrl-C stops the current task and Ctrl-D or `exit` quits. + +### Models + +`--model provider:model`, or `model:` in `gitagent.yaml`. + +| Provider | Example | Key | +|---|---|---| +| Anthropic | `anthropic:claude-opus-5-5` | `ANTHROPIC_API_KEY` | +| OpenAI (Responses API) | `openai:` | `OPENAI_API_KEY` | +| Google Gemini | `google:` | `GEMINI_API_KEY` | +| Any OpenAI-compatible endpoint | `openrouter:`, `ollama:`, `vllm:`, `:` | `{NAME}_API_KEY`; endpoint from `{NAME}_BASE_URL` (OpenRouter, Ollama and vLLM have defaults) | + +The agent's shell never sees these keys. + +### Repo mode + +`--repo ` clones into `--dir` (default: `./`). + +- **Branch:** it works on a new branch `gitagent/-` created from `--base` (default: the remote's default branch). `--branch ` continues an existing branch instead. +- **Commit:** when the agent finishes, it commits with the author from `gitagent.yaml`. Nothing is committed when nothing changed. +- **Push:** only with `--push`. +- **Safety:** it never commits to or pushes the base branch, and never force-pushes. +- **Token:** `GITAGENT_GIT_TOKEN` or `GITHUB_TOKEN` is passed to git through a helper. It is never written to the repository, the logs or the events. + +## As a library + +The `gitagent-sdk` crate offers a library interface over the same loop: `query(prompt, AgentOptions)` streams `system` / `assistant` / `user` / `result` messages, tools appear under the `.claude/` tool names (`Bash`, `Read`, `Edit`, …), and hooks are callbacks that receive `PreToolUse` / `PostToolUse` input and answer with `permissionDecision`, `updatedInput` or `updatedToolOutput`. + +```rust +use gitagent_sdk::{query, AgentOptions, AgentDefinition, HookEvent, HookMatcher, HookOutput, Message}; + +let mut q = query("Fix the failing test.", AgentOptions { + model: Some("claude-opus-5-5".into()), + allowed_tools: vec!["Bash".into(), "Read".into(), "Edit".into()], + max_turns: Some(40), + hooks: vec![HookMatcher::new(HookEvent::PreToolUse, Some("Bash"), |input| async move { + if input.tool_input["command"].as_str().unwrap_or("").contains("rm -rf") { + HookOutput::deny("no rm -rf") + } else { + HookOutput::allow() + } + })], + ..Default::default() +}).await?; +while let Some(message) = q.next().await { + if let Message::Result { result, total_cost_usd, .. } = &message { + println!("{result:?} (${total_cost_usd:.2})"); + } +} +``` + +The same shape is available from the command line: `gitagent -p "..." --output-format stream-json` prints one message per line, and `--max-turns`, `--allowed-tools`, `--disallowed-tools` and `--append-system-prompt` take the same values as `AgentOptions`. + +## Configuration + +gitagent reads `gitagent.yaml` from `--config`, else the project folder (`gitagent.yaml` or `.gitagent/gitagent.yaml`), else `~/.gitagent/gitagent.yaml`. The other files are looked up one by one in the same places (the config's folder, the project's `.gitagent/`, then `~/.gitagent/`, then the project's and home `.claude/`), and fall back to the built-in versions: + +| Path | What it is | +|---|---| +| `gitagent.yaml` | Model, turn and time limits, profile, sandbox, skills, hooks, learning, repo-mode settings | +| `prompts/system.md`, `prompts/rules.md` | The system prompt; `rules.md` is appended to it | +| `profiles/.yaml` | Per-model-family settings: tools, effort, output shaping, caching, and the optional features below | +| `skills//SKILL.md` | Instructions the agent can load on demand; their names and descriptions are listed in the system prompt | +| `hooks/pre_tool_use`, `hooks/post_tool_use` | Scripts that receive each tool call (and its output) as JSON and answer `allow`, `block` or `modify` | +| `prompts/lessons.md` | Learned lessons appended to the system prompt; unlike the other files, every layer's lessons are added up | +| `agents/.md` | Sub-agent definitions (`name`, `description`, `tools`, `disallowedTools`, `maxTurns`, `model`, `effort`; the body is the prompt), when the profile enables sub-agents | +| `.claude/agents/`, `.claude/skills/`, `.claude/settings.json` | An existing `.claude/` project layout, read as it is after the layers above: sub-agents and skills, and `PreToolUse` / `PostToolUse` hooks with their matchers, stdin JSON, exit codes and `hookSpecificOutput` answers (tool and argument names are translated both ways) | + +`profile: auto` picks `anthropic`, `openai`, `google` or `compat` from the model's provider. Every behavior that changes results is a switch in the profile. Some features stay off until measured: the environment snapshot, the stop check, the time budget, the loop detector and hiding old tool output. Context compaction is on. + +With `learning: online`, gitagent learns from each task it completes. The model reads the run's trajectory and proposes general lessons, which are saved in the project's `.gitagent/prompts/lessons.md`. A lesson is used only once two different tasks support it. Review the file like code: edit or delete lines freely. In repo mode the lessons are committed separately on the branch. `--bench` never learns and never loads project lessons. + +### Output, records and events + +- **Records:** every run writes `trajectory.json` (Harbor's ATIF format), `counters.json` and the full output of every command (`out/.log`) to `--out-dir`. They are updated after every turn. +- **Events:** `--events jsonl` prints one JSON object per line on stdout: `turn_start`, `model_text`, `tool_call`, `tool_result`, `stop_check`, `guard`, `turn_end`, `repo`, `run_end`. +- **Telemetry:** set `OTEL_EXPORTER_OTLP_ENDPOINT` to export traces and metrics (OTLP over HTTP/JSON). `OTEL_EXPORTER_OTLP_HEADERS` and `OTEL_SERVICE_NAME` are honoured. + +### Sandbox + +On Linux, outside benchmark runs, the agent's shell runs under Landlock: it may write only to the working folder, the output folder, the temp folders and `~/.cache`. A seccomp filter blocks system calls such as `mount`, loading kernel modules and `reboot`. Set `sandbox: off` in `gitagent.yaml` to disable it, or `sandbox: on` to require it. Writing under `/tests` is refused in every mode. + +## Terminal-Bench with Harbor + +```sh +scripts/build-release.sh +PYTHONPATH=$PWD harbor run -d terminal-bench/terminal-bench-2-1 \ + -a gitagent_harbor.agent:GitAgent -m anthropic/claude-opus-5-5 \ + --env-file .env --ak timeout_s=840 -k 3 -n 8 -o bench/jobs +``` + +The shim (`gitagent_harbor/agent.py`): + +- **Install:** uploads the static binary for the container's CPU and runs it with `--bench`, which uses only the built-in configuration. +- **After the run:** keeps the agent's shells open, so services it started are still up when the tests run, and returns once the agent is done. +- **Shim settings** (via `--ak`): `profile=`, `timeout_s=`. + +`bench/RESULTS.md` records every run, and `docs/INTEGRITY.md` lists the rules that keep results honest. `scripts/integrity-audit.sh` checks them. + +`python3 scripts/dashboard.py` serves a local dashboard over `bench/jobs` at http://127.0.0.1:8731, refreshing live while runs are going: jobs and trials (agent, oracle and baseline runs kept apart), accuracy per task, why each trial failed, every counter with its meaning, the guard and loop events in a trajectory, each trial's commands, outputs and log, the lesson store, the task catalog with difficulty and limits, and the feature switches of every profile. It reads only gitagent's own files, each trial's reward and the tasks' task.toml metadata. + +### Learning from finished runs + +`gitagent learn` turns finished Harbor trials into general lessons for the system prompt. It reads only gitagent's own trajectory and the reward of each trial, asks the model for lesson changes, rejects any that carry task details, and rewrites `prompts/lessons.md`. A lesson is shown to the agent only once attempts at two different tasks support it. + +```sh +gitagent learn --model anthropic:claude-opus-5-5 --from bench/jobs/ [--from …] [--dry-run] +``` + +It learns only from tasks in the `dev` list of `bench/split.json`. Review the diff of `prompts/lessons.md` before committing it; set `lessons: false` in a profile to leave lessons out. + +## Development + +```sh +cargo fmt --all +cargo clippy --all-targets -- -D warnings +cargo test --workspace +``` + +The tests need no network or keys: a scripted provider replays model replies from YAML, and the provider adapters are tested against local mock servers. + +- `BLUEPRINT.md`: what is built and why. +- `TASKS.md`: the milestones. +- `docs/DEVIATIONS.md`: where the implementation differs from the blueprint, and why. diff --git a/TASKS.md b/TASKS.md new file mode 100644 index 00000000..3614efa0 --- /dev/null +++ b/TASKS.md @@ -0,0 +1,117 @@ +# gitagent v2 — Tasks and Milestones + +Work top to bottom. Do not start a milestone until the previous one's **Done when** checks pass. Section numbers refer to `BLUEPRINT.md`. + +Scores are always: same model, Terminal-Bench 2.1 dev split, ≥3 trials, compared task by task with the previous milestone. Record every run in `bench/RESULTS.md` (date, commit, profile, model, pass rate, timeouts %, median minutes, input/output/cache tokens). + +--- + +## M0 — Measurement setup (no Rust yet) + +- [x] Install Docker, `uv`, Harbor; set API keys for two model families. +- [x] Smoke test: `harbor run -d terminal-bench/terminal-bench-2-1 -a oracle -l 5` passes. +- [ ] Baselines on the target model: Terminus 2 (`-k 3`). +- [ ] Create `bench/split.json`: 45 dev / 44 held-out task ids, stratified by category and Terminus 2 pass rate. Commit it. Never change it afterwards. +- [ ] Create `bench/RESULTS.md` with the baseline rows. + +**Done when:** baseline scores for the target model are recorded and the split is committed. + +--- + +## M1 — Skeleton: one task end to end + +- [x] Cargo workspace with crates from §5 (empty ones may be stubs); CI runs `cargo fmt --check`, `cargo clippy -D warnings`, `cargo test`. +- [x] `loader`: parse `gitagent.yaml` + `prompts/` into `AgentSpec`; load profile (§6). Unit tests with fixture configs. +- [x] `provider`: `Provider` trait + **one** adapter (target family) with native tool calling and usage incl. cache tokens. +- [x] `terminal`: one PTY session, `exec` with end marker (§5 algorithm), exit code parsing. +- [x] `core`: loop with tools `exec`, `read_file`, `task_complete`; `max_turns` limit. +- [x] `record`: ATIF trajectory appended **every turn** (follow Harbor's ATIF schema from its repo — do not invent fields); counters file. +- [x] `cli`: `--bench`, `--instruction-file`, `--model`, `--profile`, `--prompt`. +- [x] Static musl build for amd64 and arm64. +- [x] `harbor/agent.py` installed-agent shim; never touches `/tests`. +- [x] Fake-provider integration test: scripted responses run `echo hi`, then `task_complete`. + +**Done when:** one easy Terminal-Bench task passes through Harbor end to end, and CI is green. + +--- + +## M2 — Parity with Terminus 2 + +- [x] `terminal`: multiple sessions, background mode, `write_stdin` (incl. Ctrl-C `\x03`), `poll`, `kill`; yield timeout returns partial output + session id. +- [x] `provider`: second family adapter; text fallback parser; one-line repair message for malformed args; count `malformed_calls`, `repairs`. +- [x] `shaper`: full algorithm (§5) with spill files and footer; unit tests for ANSI, `\r`, collapse, cap, binary. +- [x] `tools`: `write_file` for new files; `image_read` (base64 to provider); optional `todo`. +- [x] System prompt: a short, fixed instruction sheet for our tools; byte-stable across tasks. + +**Done when:** dev score within 3 points of Terminus 2 on the same model. If far below, read failed trajectories before adding features. + +--- + +## M3 — Snapshot and stop check + +- [x] `snapshot`: probe per §5, injected into the first user message; counters `snapshot_ok/failed`. +- [x] `guards::StopCheck`: edit/verify pattern tracking; reject rules; max rejections from profile; rejection message re-shows the original instruction. +- [x] Unit tests: edit-then-complete is rejected; edit-test-complete is accepted; third attempt accepted after 2 rejections. + +**Done when:** dev score above M2 (paired), `stop_rejections` > 0 on some tasks, and fewer probing commands in first 3 turns (spot-check 10 trajectories). + +--- + +## M4 — Time, loops, context and cache + +- [x] Time budget: footer with minutes left; nudge at `nudge_at`; force stop check at `force_stop_at`. Read Harbor's task timeout from the environment/flags. +- [x] Loop detector per §5; effort bump where supported. +- [x] `context`: batched masking and structured compaction per §5; token estimates. +- [x] Cache: Anthropic breakpoints on system + last 1–2 messages; never mutate history between checkpoints; counters for cache read/write. + +**Done when:** timeouts < 10% on dev, cache read share ≥ 80% of input tokens on runs > 20 turns, and no score loss vs M3. + +--- + +## M5 — Models and edits + +- [x] `tools::apply_patch`: patch-format edit tool; Add/Update/Delete file, move; context matching. +- [x] `tools::str_replace`: exact match; clear errors for 0 or >1 matches; whitespace-tolerant fallback, counted separately. +- [x] Profiles for anthropic, openai, google, compat; `profile: auto` picks by model family. +- [x] Third provider adapter + OpenAI-compatible endpoint (test with one open-weight model). +- [ ] Tune profiles on dev only; one change at a time. +- [ ] **First held-out run**: 5 trials, target model, vs Terminus 2. + +**Done when:** each family is at or above its M4 dev score, and the held-out run beats Terminus 2 on the same model. + +--- + +## M6 — Product CLI and config + +- [x] `-p` single-prompt mode and interactive REPL on a local folder, both using the same loop as `--bench`. +- [x] Config lookup: project `gitagent.yaml`, then `~/.gitagent/`; `prompts/` and `profiles/` resolution. +- [x] Optional hooks: `pre_tool_use` / `post_tool_use` scripts, JSON in, `allow`/`block`/`modify` out; counters `hook_blocks`, `hook_modifies`. +- [x] Optional skills: index in system prompt, `skill` tool loads `SKILL.md` on demand. +- [x] `--events jsonl` event stream (BLUEPRINT §7). +- [x] Repo mode (BLUEPRINT §5): `--repo` clones the repo, creates `gitagent/-` from the base branch (or resumes `--branch`), runs the loop in it, commits when the stop check accepts, pushes only with `--push`. +- [x] Token handling: read from `GITAGENT_GIT_TOKEN` or `GITHUB_TOKEN`, pass via `GIT_ASKPASS`; test that the token never appears in `.git/config`, logs, events or trajectories. +- [x] Safety tests: refuses to run on the base branch; never force-pushes; no commit when nothing changed; resume continues on the same branch. +- [x] Repo mode is disabled in `--bench`. + +**Done when:** a user can run `gitagent -p "fix the failing test" --dir ./project` and `gitagent --repo -p "fix the failing test" --push` on a real project (the second leaves a new branch with the fix pushed), and the benchmark score is unchanged (these features are off in `--bench`). + +--- + +## M7 — Security, telemetry, release + +- [x] `sandbox`: Landlock (write only to workspace, spill dir, tmp), seccomp deny-list; on by default outside `--bench`; `/tests` writes denied always. +- [x] OpenTelemetry spans and metrics (turns, tools, tokens, cost), off unless `OTEL_EXPORTER_OTLP_ENDPOINT` is set. +- [x] Integrity audit of profiles and prompts: nothing task-specific, no task names, no verifier references. +- [ ] Final held-out run (5 trials) on two model families; publish results with public trajectories. +- [x] README, install script, release binaries for Linux amd64/arm64 and macOS. + +**Done when:** the release checklist passes and held-out results are published. + +--- + +## Later (only when needed) + +- Open a pull request from the working branch (GitHub API). +- MCP client (`rmcp`). +- SDK (TypeScript or Python) over `--events jsonl`. +- Optional fresh-context auditor, only if it wins on dev under default timeouts. diff --git a/bench/RESULTS.md b/bench/RESULTS.md new file mode 100644 index 00000000..2fcc2c44 --- /dev/null +++ b/bench/RESULTS.md @@ -0,0 +1,32 @@ +# Benchmark results + +Terminal-Bench 2.1. Dev split unless the Split column says otherwise. Same model within a comparison, ≥3 trials, compared task by task with the previous milestone. Held-out runs only at M5 and release. + +| Date | Commit | Agent | Profile | Model | Split | Trials | Pass rate | Timeouts % | Median min | Input tok | Output tok | Cache read tok | Notes | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 2026-10-14 | — | oracle | — | — | smoke (`-l 5`) | 1 | 2/5 | 0% (1 verifier timeout) | — | — | — | — | macOS arm64 host; all task images are amd64 and ran under emulation. Not a valid benchmark host. | +| 2026-10-15 | — | oracle | — | — | same 5 tasks | 1 | 4/5 | 0% | — | — | — | — | macOS host, Colima `bench` VM (vz, 8 CPU, 14 GB), amd64 via QEMU (Rosetta not installed yet). 2/5 → 4/5 from resources alone. The failure is a compute-heavy task whose checker ran 11.6 min; rerun with Rosetta pending. | +| 2026-10-14 | a91af9d | gitagent | built-in:anthropic | anthropic/claude-opus-5-5 | single task `fix-git` (M1 end-to-end check) | 1 | 1/1 | 0% | 0.6 | 17840 | 828 | 12610 | macOS host under emulation; plumbing check, not a score. `fix-git` trajectory was read, so it must go in the dev split. | +| 2026-10-14 | b6bec94 | gitagent | built-in:anthropic | anthropic/claude-opus-5-5 | single task `fix-git` (M2 stack check) | 1 | 1/1 | 0% | — | 37645 | 1397 | 31204 | macOS host under emulation; plumbing check, not a score. 9 turns: one wasted on a closing summary without task_complete (prompt fixed in 35b65ec). | +| 2026-10-14 | 10aa140+M3 | gitagent | bench/profiles/anthropic-m3.yaml | anthropic/claude-opus-5-5 | single task `fix-git` (M3 stack check) | 1 | 1/1 | 0% | — | 17054 | 613 | 11405 | macOS host under emulation; plumbing check, not a score. Snapshot ok, stop check accepted; 4 turns (6 and 9 in the earlier runs). | +| 2026-10-14 | 914a4a5+M4 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | single task `fix-git` (M4 stack check) | 1 | 1/1 | 0% | — | 27478 | — | 21291 | macOS host under emulation; plumbing check, not a score. API accepted server-side clearing with the other options; footer showed minutes left. | +| 2026-10-14 | b9e69d0 | gitagent | built-in:compat | ollama/qwen2.5:1.5b (local) | one toy task (compat adapter check) | 2 | 1/2 completed, 2/2 file correct | 0% | — | — | — | — | M5 open-weight check on macOS via Ollama's OpenAI-compatible endpoint; not a Terminal-Bench run. The failed run never called task_complete (small model). | +| 2026-10-14 | 883a471 | gitagent | built-in:anthropic | anthropic/claude-opus-5-5 | M6 done-when check on a small project (not Terminal-Bench) | 2 | 2/2 | 0% | — | — | — | — | `-p "fix the failing test" --dir` fixed it in 4 turns; `--repo -p ... --push` pushed gitagent/fix-the-failing-test-* with a one-line fix; main untouched. | +| 2026-10-15 | aab5366 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | single task `kv-store-grpc` (medium) | 1 | 1/1 | 0% | 0.8 | 10626 | 1262 | 6314 | macOS host under emulation; not a score. 3 turns; server left running for the tests via --linger. Trajectory read: `kv-store-grpc` must go in the dev split. | +| 2026-10-15 | 12b67d9 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev pass 1: 10 tasks × 1 trial (1 easy, 6 medium, 3 hard) | 10 | 5/10 scored (5/10 of all) | 0% (1 crash) | 2.5 | 1551315 | 46169 | 1425656 | macOS Colima VM (8 CPU, QEMU amd64); dev-only, not a score. Cache hit 91%. Rejected after task_complete: build-cython-ext, make-doom-for-mips, raman-fitting, mailman; each left something unverified (a known failing test, an unopened output image, a traceback in a service log, an assumed unit convention). headless-terminal: gitagent exited 139 (segfault) at startup before writing anything; not reproduced by hand in the same image, suspected QEMU user-mode flake. Learner run on all 9 trajectories: 9 lessons added (12 stored), 3 proposals rejected for length, 0 promoted (each backed by one task). | +| 2026-10-15 | 749e58f | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev pass 2: 10 tasks × 1 trial (1 easy, 6 medium, 3 hard) | 10 | 7/10 | 0% | 2.7 | 3413233 | 105203 | 2997048 | macOS Colima VM (QEMU amd64); dev-only, not a score. Est. cost $4.78 (bench/pricing.json). Rejected after task_complete: fix-ocaml-gc, qemu-alpine-ssh, tune-mjcf. headless-terminal rerun from pass 1 passed (7 turns) after the shim's relaunch fix. Dev accuracy by task after passes 1–2: 15/22 (easy 3/3, medium 8/13, hard 4/6). | +| 2026-10-15 | — | oracle | — | — | the 3 tasks gitagent failed in pass 2 (fix-ocaml-gc, qemu-alpine-ssh, tune-mjcf) | 1 | 0/3 | 0% | — | — | — | — | Machine check on the Colima `bench` VM (QEMU amd64, no Rosetta): the reference solutions fail too (no AVX, nested VM does not boot, gcc segfaults under emulation). Those three pass-2 failures are the machine's, not the agent's; they must be rerun once the host runs amd64 faithfully. | +| 2026-10-15 | — | oracle | — | — | the 4 tasks the oracle failed under QEMU (schemelike-metacircular-eval, fix-ocaml-gc, qemu-alpine-ssh, tune-mjcf) | 1 | 3/4 | 0% | — | — | — | — | Machine check after installing Rosetta 2 (bench VM, `rosetta: true`; the CPU test runs 7× faster than under QEMU). Three now pass; qemu-alpine-ssh still fails because it boots a VM inside the VM, which this host cannot do. From here on, failures on tasks other than qemu-alpine-ssh are the agent's. | +| 2026-10-15 | e581024 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev pass 3: 10 tasks × 1 trial (6 medium, 3 hard, 1 easy) | 10 | 10/10 | 0% | 2.0 | 658774 | 46105 | 568543 | First pass under Rosetta (machine 7× faster) and the first with learned lessons in the prompt (5 at the start, 9 by the end). Est. cost $1.49. One trial was lost to a stale aarch64 binary (the image is multi-arch) and rerun after rebuilding both targets; the shim now refuses a stale sibling binary. Dev accuracy by task on runnable tasks after passes 1–3: 26/31. Faster machine, new lessons and single runs are confounded; a lessons on/off comparison is still to do. | +| 2026-10-15 | f4a0a30 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev pass 4: the remaining 14 tasks × 1 trial | 14 | 10/14 | 0% | 2.9 | 3758587 | 103753 | 3546921 | Rosetta; 9–10 learned lessons in the prompt. Est. cost $3.84. Rejected after task_complete: mteb-retrieve, protein-assembly, sparql-university (ambiguous specs read differently from the checker; no admitted-unverified cases). mteb-leaderboard's container timed out at start once (Harbor EnvironmentStartTimeoutError) and passed on a rerun; fix-ocaml-gc from pass 2 passed on its Rosetta rerun (16 turns). First full pass over the dev split done: 37/44 on runnable tasks (easy 3/3, medium 22/27, hard 12/15); qemu-alpine-ssh cannot run on this host. Total est. cost of passes 1–4 $12.89. | +| 2026-10-15 | 480265d | oracle | — | — | all 89 tasks (dev + held-out; the oracle does not learn) | 89 | 83/89 | 0% | — | — | — | — | First run on optimus (native x86_64 Linux, Docker 29, `-n 12`). Failed: build-cython-ext, feal-differential-cryptanalysis, mcmc-sampling-stan, protein-assembly, qemu-alpine-ssh, qemu-startup. Reruns (`optimus-oracle-retry`, 8 CPUs each; `optimus-oracle-retry2`): feal-differential-cryptanalysis passed (flaky); the other five failed again. Only rewards were read. caffe-cifar-10 passed after its dataset download stalled ~25 min (slow external host). Dev tasks the oracle cannot pass here: build-cython-ext, mcmc-sampling-stan, protein-assembly, qemu-alpine-ssh; agent failures on them are not counted against the agent. | +| 2026-10-15 | — | terminus-2 | — | anthropic/claude-opus-5-5 | dev, 44 tasks (all but qemu-alpine-ssh) × 1 trial; stopped by the owner with 41 scored | 41 | 36/41 | 10% (4 AgentTimeoutError) | 2.5 | 151M | 0.2M | 47M | Baseline on optimus. Paired with gitagent's latest run on the 39 runnable tasks both ran: 34 vs 34. **Cost ≈ $368 at Opus 5.5 list prices (bench/pricing.json; LiteLLM reported $455 because it prices Opus 5.5 at the older Opus 5 rates), almost all from two runaway loops:** on password-recovery (489 calls, $44) and crack-7z-hash (1,316 calls in 28 min, 77M input tokens, $304) the model returned empty replies from the first call (most likely a refusal) and Terminus re-sent the whole growing conversation asking for valid JSON until the time limit. protein-assembly (29M tokens, cancelled) did the same with caching. Without those three, ≈ $14. Cancelled when stopped: crack-7z-hash, install-windows-3.11, protein-assembly. | +| 2026-10-16 | 480265d | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev, all 45 tasks × 1 trial (round r1, `scripts/run-dev-pass.sh r1`) | 45 | 43/45 (95.6%) | 0% | 1.1 | 9.41M | 0.31M | 8.66M | First full dev pass on optimus (native amd64), code unchanged since the Mac passes. Est. cost $11.73 ($0.26 per task); 92% of input served from the cache. Failed: qemu-alpine-ssh (the machine cannot run it: the oracle fails it too and no harness has passed it) and filter-js-from-html (declared done after testing 67 XSS payloads in headless Chromium; most likely failed the checker on "do not alter the formatting", since the filter rewrites attribute quotes on tags it cleans; it passed in r0 and for Terminus). build-cython-ext, mcmc-sampling-stan and protein-assembly passed although the oracle fails all three here, so their reference solutions are broken, not the machine. Against Terminus 2 on the 41 tasks both scored: 97.6% vs 87.8% (+9.8 points; 5 won, 1 lost), median agent time 65 s vs 151 s, cost per run $0.23 vs $1.37. Single runs; the blueprint scores with ≥3. Found: the /tests write guard refused a read-only `ls /tests` (one wasted turn). | +| 2026-10-16 | 480265d | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev, all 45 tasks × 3 trials (round r1 = the run above plus `K=2 scripts/run-dev-pass.sh r1-rep`) | 135 | 127/135; mean per-task 94.1% (96.2% without qemu-alpine-ssh) | 0% | — | — | — | — | Round r1 with three runs per task, code unchanged. 41 tasks passed all three runs, 43 at least once. Always failed: qemu-alpine-ssh (machine) and filter-js-from-html (0/3: every run kept clean files byte-identical instead of the parser-normalised output the instruction allows; the r0 run that passed always wrote the normalised form). Failed once in three: protein-assembly, make-doom-for-mips. Against Terminus 2 on the 41 tasks it scored: 96.7% vs 87.8% (+8.9 points; Terminus 1 run per task). Est. cost $33.36 for 135 trials. Lessons learned from all 135 trials afterwards (b15bb30, 98b89c4). | +| 2026-10-16 | 97fc537 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev, all 45 tasks × 1 trial (round r2) | 45 | 42/45 (93.3%; 95.5% without qemu-alpine-ssh) | 0% | 1.2 | — | — | — | r1 plus the /tests guard fix (f4c6e82) and lessons relearned from r1's 135 trials (b15bb30, 98b89c4). Same as r1 (94.1%) within noise: only make-doom-for-mips (2/3 in r1 → pass) and protein-assembly (2/3 in r1 → fail) differ. Guard fix confirmed: 0 false /tests refusals against 20 in r1's 135 trials. 14.8 turns per trial against 13.0, median 74 s against 82 s, $0.266 per trial against $0.247; 93.1% of input from the cache. Failed: qemu-alpine-ssh (machine), filter-js-from-html (0/4 on optimus), protein-assembly. Est. cost $11.98. | +| 2026-10-16 | 97fc537 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | **held-out, all 44 tasks × 3 trials** (r2's code frozen; first use of the held-out split) | 132 | **126/132; mean per-task 95.5%** (97.7% without qemu-startup); single runs 95.5% / 93.2% / 97.7% | 0% | — | — | — | — | The honest score: tasks never run, read or learned from before. Not lower than dev (r1 94.1%, r2 93.3%), so tuning on dev did not overfit. 40/44 tasks passed all three runs, 43 at least once. Not always passed: qemu-startup 0/3 (machine), modernize-scientific-stack, video-processing, extract-moves-from-video 2/3 each. Published Terminal-Bench 2.1 results for comparison (all 89 tasks): Artificial Analysis top 91.4% (Fable 5.1, max effort), Vals.ai Opus 5.5 + Terminus 2 87.64%, Snorkel top 87.4% ±0.9 (GPT-6 Astra + Codex CLI). Est. cost $39.01 ($0.30 per trial). Run note: 15 trials first failed to start (Docker ran out of network address pools at 32 concurrent trials), then failed again on resume because `harbor job resume` does not reload `--env-file`; they were re-run with the key exported and only those complete runs are counted. | +| 2026-10-17 | 57f0dc2 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | dev, all 45 tasks × 3 trials (round r3) | 135 | 124/135 (91.9%; 93.9% without qemu-alpine-ssh); 39/45 tasks passed all three runs | 0% | 1.4 | 27.3M | 906k | 25.4M | r2's harness with the system prompt and apply_patch tool written anew (same rules, new wording; apply_patch is not used by this profile), same lessons (98b89c4). Same as r1 (94.1%, same 135-trial design) within noise: no task flipped; the gap is three single lost runs on tasks that were 3/3 (dna-assembly, polyglot-c-py, raman-fitting) plus protein-assembly 1/3 (2/3 in r1). Failed trials ended normally (11–23 turns, no timeouts or loops). 13.2 turns per trial against 13.0, median 82 s in both, $0.243 per trial against $0.247; 92.9% of input from the cache; 0 false /tests refusals. Failed: qemu-alpine-ssh (machine), filter-js-from-html (0/3, unchanged since r0). Est. cost $32.86. | +| 2026-10-17 | 57f0dc2 | gitagent | bench/profiles/anthropic-m4.yaml | anthropic/claude-opus-5-5 | held-out, all 44 tasks × 1 trial (check of the rewritten prompt, round heldout-r3) | 44 | 40/44 (90.9%; 93.0% without qemu-startup) | 0% | — | — | — | — | Same code as r3. One trial below the first held-out run's single-run band (93.2 / 95.5 / 97.7%): failed qemu-startup (machine), extract-moves-from-video and video-processing (2/3 before) and dna-insert (3/3 before). Alone it is noise (p = 0.22); pooled with r3 the rewritten prompt stands at 164/179 (91.6%) against 253/267 (94.8%) for the old one, p = 0.13. A sanity check, not a score; not used to change code. Est. cost $14.08. | +| 2026-10-17 | 57f0dc2 | gitagent | bench/profiles/openai-m4.yaml | openai/gpt-6-astra | dev, all 45 tasks × 1 trial (round g1, first GPT-6 round) | 45 | 40/45 (88.9%; 90.9% without qemu-alpine-ssh) | 0% | 2.4 | — | — | — | Same harness build and lessons as r3; the OpenAI profile with the M4 switches on and apply_patch as the edit tool. Against Opus 5.5 on the same tasks: r2 93.3% (1 run), r3 91.9% (3 runs). Failed the two tasks every run fails plus mailman, make-doom-for-mips and mteb-retrieve; passed protein-assembly. 22.6 turns per trial, median 141 s, $1.30 per trial (median $0.66); 33 stop-check rejections, no timeouts or loops. apply_patch: 34 calls, 0 failures. pytorch-model-cli's checker timed out once (Harbor), re-run and passed. Untuned first round. Est. cost $58.60. | +| 2026-10-17 | 9775813 | gitagent | bench/profiles/openai-m4.yaml | openai/gpt-6-astra | dev, all 45 tasks × 1 trial (round g2) | 45 | 40/45 (88.9%; 90.9% without qemu-alpine-ssh) | 0% | 2.1 | — | — | — | g1 plus lessons relearned from r3 and g1 (180 trials; 44 added, 24 edited, 151 stored, 10 shown). Same accuracy as g1: make-doom-for-mips now passes, dna-assembly now fails. Cleaner finishing: stop-check rejections 31 → 18, turns 22.5 → 21.4, median 141 s → 126 s, $1.20 per trial. apply_patch 30 calls, 1 failure. Est. cost $54.12. | +| 2026-10-17 | 9775813 | gitagent | bench/profiles/openai-m4.yaml | openai/gpt-6-astra | held-out, all 44 tasks × 1 trial (round heldout-g2, g2's code frozen) | 44 | 37/44 (84.1%; 86.0% without qemu-startup; 88.1% of the 42 tasks the API let it attempt; 90.2% also leaving out qemu-startup) | 0% | — | — | — | — | Two runs ended in an OpenAI cyber_policy refusal (break-filter-js-from-html, vulnerable-secret): content flagged as a possible cybersecurity risk, so the tasks were not attempted. Other failures: qemu-startup (machine), video-processing, and extract-elf, sanitize-git-repo, torch-pipeline-parallelism (all 3/3 for Opus). Against Opus 5.5 on the same tasks: 95.5% (3 runs, old prompt), 90.9% (1 run, current prompt). One run; not used to change code. Est. cost $51.72. | diff --git a/bench/fixtures/smoke.yaml b/bench/fixtures/smoke.yaml new file mode 100644 index 00000000..d45d3f5b --- /dev/null +++ b/bench/fixtures/smoke.yaml @@ -0,0 +1,12 @@ +# Scripted dry run for checking the Harbor integration without a model API. +# It only looks around and then stops; it does not try to solve the task. +- text: Looking at the environment. + tool_calls: + - name: exec + arguments: {cmd: "pwd && ls -la | head -20 && uname -m"} +- tool_calls: + - name: read_file + arguments: {path: /etc/os-release, limit: 5} +- tool_calls: + - name: task_complete + arguments: {summary: Integration dry run.} diff --git a/bench/leaderboard.json b/bench/leaderboard.json new file mode 100644 index 00000000..1d0e7c89 --- /dev/null +++ b/bench/leaderboard.json @@ -0,0 +1,112 @@ +{ + "_note": "Published Terminal-Bench 2.1 results used as reference points on the dashboard's Rounds page. All are on the full 89-task set; our dev rounds cover 45 tasks we tuned on, so they are not directly comparable until the held-out run. Update by hand when the leaderboards change.", + "entries": [ + { + "source": "Vals.ai", + "url": "https://www.vals.ai/benchmarks/terminal-bench-2-1", + "date": "2026-09-28", + "harness": "Terminus 2", + "model": "Claude Opus 5.5", + "accuracy": 87.64, + "note": "pass@1" + }, + { + "source": "Vals.ai", + "url": "https://www.vals.ai/benchmarks/terminal-bench-2-1", + "date": "2026-09-28", + "harness": "Terminus 2", + "model": "GPT-6 Astra", + "accuracy": 87.27, + "note": "pass@1" + }, + { + "source": "Vals.ai", + "url": "https://www.vals.ai/benchmarks/terminal-bench-2-1", + "date": "2026-09-28", + "harness": "Terminus 2", + "model": "GPT-5.6 Sol", + "accuracy": 85.77, + "note": "pass@1" + }, + { + "source": "Vals.ai", + "url": "https://www.vals.ai/benchmarks/terminal-bench-2-1", + "date": "2026-09-28", + "harness": "Terminus 2", + "model": "Claude Fable 5.1", + "accuracy": 85.02, + "note": "pass@1" + }, + { + "source": "Vals.ai", + "url": "https://www.vals.ai/benchmarks/terminal-bench-2-1", + "date": "2026-09-28", + "harness": "Terminus 2", + "model": "Claude Opus 5", + "accuracy": 84.64, + "note": "pass@1" + }, + { + "source": "Vals.ai", + "url": "https://www.vals.ai/benchmarks/terminal-bench-2-1", + "date": "2026-09-28", + "harness": "Claude Code", + "model": "Claude Opus 4.8", + "accuracy": 69.66, + "note": "pass@1" + }, + { + "source": "Snorkel (official leaderboard)", + "url": "https://snorkel.ai/leaderboard/terminal-bench-2-1/", + "date": "2026-09-03", + "harness": "Codex CLI", + "model": "GPT-6 Astra", + "accuracy": 87.4, + "error": 0.9 + }, + { + "source": "Snorkel (official leaderboard)", + "url": "https://snorkel.ai/leaderboard/terminal-bench-2-1/", + "date": "2026-06-09", + "harness": "Claude Code", + "model": "Claude Fable 5", + "accuracy": 83.8, + "error": 1.2 + }, + { + "source": "Snorkel (official leaderboard)", + "url": "https://snorkel.ai/leaderboard/terminal-bench-2-1/", + "date": "2026-04-23", + "harness": "Codex CLI", + "model": "GPT-5.5", + "accuracy": 83.1, + "error": 1.1 + }, + { + "source": "Snorkel (official leaderboard)", + "url": "https://snorkel.ai/leaderboard/terminal-bench-2-1/", + "date": "2026-06-09", + "harness": "Terminus 2", + "model": "Claude Fable 5", + "accuracy": 80.4, + "error": 1.2 + }, + { + "source": "Snorkel (official leaderboard)", + "url": "https://snorkel.ai/leaderboard/terminal-bench-2-1/", + "date": "2026-05-28", + "harness": "Claude Code", + "model": "Claude Opus 4.8", + "accuracy": 78.9, + "error": 1.3 + }, + { + "source": "Artificial Analysis", + "url": "https://artificialanalysis.ai/evaluations/terminalbench-2-1", + "date": "2026-10", + "harness": "Artificial Analysis's own harness", + "model": "Claude Fable 5.1 (Max effort)", + "accuracy": 91.4 + } + ] +} diff --git a/bench/lessons/lessons.json b/bench/lessons/lessons.json new file mode 100644 index 00000000..1d213569 --- /dev/null +++ b/bench/lessons/lessons.json @@ -0,0 +1,3216 @@ +{ + "next_id": 151, + "lessons": [ + { + "id": 1, + "text": "When work seems lost in a version-controlled project, look in the history logs that record every HEAD movement (such as the reflog) and in the stash before assuming it is gone, because commits made on a detached or temporary checkout usually still exist there.", + "importance": 2, + "helpful": 10, + "harmful": 0, + "tasks": [ + "fix-git" + ], + "sources": [ + "m1-fix-git/fix-git__rznNHc5", + "m2-fix-git/fix-git__oGtXbYW", + "m3-fix-git/fix-git__D3yv22h", + "m4-fix-git/fix-git__h7ah2xR", + "r1-t900/fix-git__tPmQCGY", + "r1-rep-t900/fix-git__EWnZRi7", + "r1-rep-t900/fix-git__Fr6g7vD", + "r3-t900/fix-git__4kuz8Bf", + "r3-t900/fix-git__97bA6rv", + "r3-t900/fix-git__EVT5tcy", + "g1-t900/fix-git__K4oagw6" + ] + }, + { + "id": 2, + "text": "Before resolving a merge conflict, diff each side against the common ancestor to see what each one intended, and do not assume either side is correct: test the resolved result against the spec or examples, then confirm that no conflict markers remain and that the merge is committed.", + "importance": 3, + "helpful": 17, + "harmful": 0, + "tasks": [ + "fix-git", + "merge-diff-arc-agi-task" + ], + "sources": [ + "m1-fix-git/fix-git__rznNHc5", + "m2-fix-git/fix-git__oGtXbYW", + "m3-fix-git/fix-git__D3yv22h", + "m4-fix-git/fix-git__h7ah2xR", + "pass3-1/merge-diff-arc-agi-task__NtQSVZB", + "r1-t900/fix-git__tPmQCGY", + "r1-t900/merge-diff-arc-agi-task__2rxMpAh", + "r1-rep-t900/fix-git__EWnZRi7", + "r1-rep-t900/fix-git__Fr6g7vD", + "r1-rep-t900/merge-diff-arc-agi-task__SG6sUoA", + "r1-rep-t900/merge-diff-arc-agi-task__rrAgxvw", + "r3-t900/fix-git__4kuz8Bf", + "r3-t900/fix-git__97bA6rv", + "r3-t900/fix-git__EVT5tcy", + "g1-t900/fix-git__K4oagw6", + "r3-t900/merge-diff-arc-agi-task__Eg7roT5", + "r3-t900/merge-diff-arc-agi-task__LaYRW7y", + "r3-t900/merge-diff-arc-agi-task__jN5ZQCk", + "g1-t900/merge-diff-arc-agi-task__Wicye3a" + ] + }, + { + "id": 3, + "text": "When a task requires a service to keep running, launch it detached from the shell with its output redirected to a log, then confirm it works by making a real client request against it, including an edge case such as a missing key, before declaring the task complete.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "kv-store-grpc" + ], + "sources": [ + "task1-kv-store-grpc/kv-store-grpc__Sj4AjYz", + "r1-t900/kv-store-grpc__vuLhgEt", + "r1-rep-t900/kv-store-grpc__7ynYVuN", + "r1-rep-t900/kv-store-grpc__H5mcipG", + "r3-t900/kv-store-grpc__iLR9Ty8", + "r3-t900/kv-store-grpc__jnvKq3w", + "r3-t900/kv-store-grpc__oMLDERL", + "g1-t900/kv-store-grpc__mf7amCP" + ] + }, + { + "id": 4, + "text": "When the task says the project's tests must pass, run the whole required suite as early as possible without stop-at-first-failure flags, so every failure shows up at once while there is still time to fix them; never mark the task complete while a required test is still failing, even if you think the failure is outside the main scope.", + "importance": 3, + "helpful": 3, + "harmful": 0, + "tasks": [ + "build-cython-ext", + "fix-ocaml-gc" + ], + "sources": [ + "pass1-a/build-cython-ext__LsXpBxY", + "pass2-d/fix-ocaml-gc__gefTDzu", + "pass2-retry-ocaml/fix-ocaml-gc__saSjV9F", + "r1-t900/build-cython-ext__eVWTonP" + ] + }, + { + "id": 5, + "text": "Under a tight time budget, start slow work (installs, builds, a timed reference run of the original) in the background at once, logging output and taking stdin from /dev/null, and keep developing meanwhile. Install only what is needed, and poll the log and job state instead of waiting blindly.", + "importance": 17, + "helpful": 47, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler", + "build-cython-ext", + "build-pov-ray", + "compile-compcert", + "crack-7z-hash", + "fix-ocaml-gc", + "gcode-to-text", + "install-windows-3.11", + "kv-store-grpc", + "mcmc-sampling-stan", + "merge-diff-arc-agi-task", + "polyglot-c-py", + "pytorch-model-cli", + "query-optimize", + "raman-fitting", + "sqlite-with-gcov" + ], + "sources": [ + "pass1-a/build-cython-ext__LsXpBxY", + "pass2-a/pytorch-model-cli__8ikPT2P", + "pass3-4/build-pov-ray__y3tJCzZ", + "pass3-1/adaptive-rejection-sampler__vNJP6CU", + "pass3-1/merge-diff-arc-agi-task__NtQSVZB", + "pass4-5/install-windows-3.11__cD5fPWQ", + "r1-t900/adaptive-rejection-sampler__PArvFEg", + "r1-t12000/build-pov-ray__Mu92oJj", + "r1-t3600/fix-ocaml-gc__uj5TJYh", + "r1-t900/gcode-to-text__tyJktT9", + "r1-t3600/install-windows-3.11__osH4pRk", + "r1-t1800/mcmc-sampling-stan__YoaMuqy", + "r1-t900/merge-diff-arc-agi-task__2rxMpAh", + "r1-t900/pytorch-model-cli__SnGhQgT", + "r1-rep-t900/adaptive-rejection-sampler__eh3usHZ", + "r1-rep-t900/build-cython-ext__2VMc5JK", + "r1-rep-t12000/build-pov-ray__Q876nYw", + "r1-rep-t1800/mcmc-sampling-stan__6M43xXY", + "r1-rep-t1800/mcmc-sampling-stan__ogWSMmh", + "r1-rep-t900/pytorch-model-cli__R67cSvk", + "r1-rep-t900/pytorch-model-cli__rkY9xfq", + "r1-rep-t900/query-optimize__9jCUzwb", + "r1-rep-t900/raman-fitting__3YhzPND", + "r1-rep-t900/sqlite-with-gcov__cTBWbG5", + "r3-t900/adaptive-rejection-sampler__LrDGcCP", + "r3-t900/adaptive-rejection-sampler__j3gYRYy", + "r3-t900/adaptive-rejection-sampler__xezMuBL", + "g1-t900/adaptive-rejection-sampler__SPEQCFZ", + "g1-t900/build-cython-ext__GyNHJk2", + "r3-t12000/build-pov-ray__Qo6eLfg", + "r3-t2400/compile-compcert__3FUpnGN", + "r3-t2400/compile-compcert__8y9sHD7", + "r3-t2400/compile-compcert__ho6QX8C", + "r3-t1800/crack-7z-hash__MbUQjgc", + "g1-t3600/fix-ocaml-gc__EXTWQgd", + "g1-t900/kv-store-grpc__mf7amCP", + "r3-t1800/mcmc-sampling-stan__2grZFD7", + "r3-t1800/mcmc-sampling-stan__LE2RGm3", + "r3-t1800/mcmc-sampling-stan__bkVPVFd", + "g1-t900/merge-diff-arc-agi-task__Wicye3a", + "r3-t900/polyglot-c-py__PRGhezc,r3-t900/polyglot-c-py__nMBTWWS", + "r3-t900/polyglot-c-py__p8j9Tm8", + "r3-t900/pytorch-model-cli__mB9QiSE", + "g1-t900/pytorch-model-cli__qHP6PuT", + "r3-t900/query-optimize__J7Lnk9f", + "r3-t900/query-optimize__TiunRY9", + "r3-t900/raman-fitting__gbKShkf", + "r3-t900/sqlite-with-gcov__JKx9qun", + "r3-t900/sqlite-with-gcov__aTQSY3F", + "g1-t900/sqlite-with-gcov__C3fP2Et" + ] + }, + { + "id": 6, + "text": "When a hidden harness will call your function or run your deliverable, write your own driver that invokes it the same way across many random inputs plus edge cases (zero, extremes, boundaries), compares against a simple reference, and times it, before declaring completion.", + "importance": 6, + "helpful": 14, + "harmful": 0, + "tasks": [ + "circuit-fibsqrt", + "feal-differential-cryptanalysis", + "headless-terminal", + "merge-diff-arc-agi-task", + "polyglot-rust-c" + ], + "sources": [ + "pass1-b/feal-differential-cryptanalysis__6H9BX99", + "pass1-retry/headless-terminal__nYNzBHJ", + "pass3-3/circuit-fibsqrt__eLneuTd", + "r1-t3600/circuit-fibsqrt__66WdFy2", + "r1-t1800/feal-differential-cryptanalysis__Hr4kPkt", + "r1-rep-t3600/circuit-fibsqrt__DNQpDEy", + "r1-rep-t3600/circuit-fibsqrt__DyUFemQ", + "r1-rep-t1800/feal-differential-cryptanalysis__XGgc5Fe", + "r1-rep-t1800/feal-differential-cryptanalysis__YQAZFGm", + "r1-rep-t900/merge-diff-arc-agi-task__rrAgxvw", + "r3-t3600/circuit-fibsqrt__HiEsydF", + "r3-t3600/circuit-fibsqrt__eP6oWWJ", + "r3-t3600/circuit-fibsqrt__tLnVfqd", + "g1-t3600/circuit-fibsqrt__vquinqL", + "r3-t1800/feal-differential-cryptanalysis__FucyjUu", + "g1-t900/polyglot-rust-c__sQrHBkV" + ] + }, + { + "id": 7, + "text": "When a solution works by filtering a candidate space down to one answer, check whether distinct candidates could be indistinguishable under your test (equivalent or symmetric solutions), and confirm the surviving answer is unique rather than trusting that the first survivor is right.", + "importance": 3, + "helpful": 4, + "harmful": 0, + "tasks": [ + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis" + ], + "sources": [ + "pass1-b/feal-differential-cryptanalysis__6H9BX99", + "pass3-2/feal-linear-cryptanalysis__LNvJA8x", + "r1-t1800/feal-differential-cryptanalysis__Hr4kPkt", + "r1-rep-t1800/feal-differential-cryptanalysis__XGgc5Fe", + "r3-t1800/feal-differential-cryptanalysis__FucyjUu" + ] + }, + { + "id": 8, + "text": "Inspect every artifact shipped with the task inputs (binaries, scripts, leftover sources), since a generator or reference is often present. Run it in a scratch dir to confirm it reproduces the given output exactly, then recover its logic via strings, symbols and disassembly rather than guessing.", + "importance": 5, + "helpful": 10, + "harmful": 0, + "tasks": [ + "crack-7z-hash", + "make-mips-interpreter", + "password-recovery", + "path-tracing" + ], + "sources": [ + "pass1-b/path-tracing__wSyxsuh", + "pass4-3/make-mips-interpreter__c8DW2ck", + "pass4-1/password-recovery__WRY4VPe", + "r1-t1800/path-tracing__HJxWpkM", + "r1-rep-t1800/crack-7z-hash__6WFusMK", + "r1-rep-t1800/path-tracing__E39pVqQ", + "r1-rep-t1800/path-tracing__GugRcxU", + "r3-t1800/crack-7z-hash__MbUQjgc", + "r3-t1800/path-tracing__3P8c5eJ", + "r3-t1800/path-tracing__AZ2NaW7", + "r3-t1800/path-tracing__g32vhrn", + "g1-t1800/path-tracing__hH2VFLW" + ] + }, + { + "id": 9, + "text": "Before building or running anything that might write over provided input files, back them up and work in a scratch directory, then reproduce the stated acceptance check locally (the exact build command, the exact size or format limits, and the scoring metric against the reference) before declaring completion.", + "importance": 7, + "helpful": 16, + "harmful": 0, + "tasks": [ + "db-wal-recovery", + "distribution-search", + "gpt2-codegolf", + "overfull-hbox", + "path-tracing", + "query-optimize" + ], + "sources": [ + "pass1-b/path-tracing__wSyxsuh", + "pass2-d/distribution-search__LUVHzza", + "pass2-b/overfull-hbox__BbekRBh", + "pass3-1/query-optimize__E4BccaN", + "pass4-1/db-wal-recovery__jgiY4np", + "r1-t900/db-wal-recovery__qnwnRjY", + "r1-t750/overfull-hbox__ee9c4jM", + "r1-t1800/path-tracing__HJxWpkM", + "r1-rep-t900/db-wal-recovery__AimMLx5", + "r1-rep-t900/db-wal-recovery__jkQmU3W", + "r1-rep-t900/gpt2-codegolf__iWbwUmN", + "r1-rep-t900/gpt2-codegolf__mzFmpW3", + "r1-rep-t1800/path-tracing__GugRcxU", + "r3-t900/db-wal-recovery__GwatioF", + "r3-t900/db-wal-recovery__vPbhdHk", + "r3-t900/db-wal-recovery__xtS6cXe", + "r3-t1800/path-tracing__g32vhrn" + ] + }, + { + "id": 10, + "text": "When filling in placeholders such as admitted proofs, TODO stubs or skipped tests, verify both that the artifact builds and that no escape hatch remains, using the tool's own dependency or assumption report where one exists, rather than trusting a clean compile alone.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "prove-plus-comm" + ], + "sources": [ + "pass1-a/prove-plus-comm__iSrWbGV", + "r1-t900/prove-plus-comm__W6wjt7b", + "r1-rep-t900/prove-plus-comm__48FaXoU", + "r1-rep-t900/prove-plus-comm__KpDxmYD", + "r3-t900/prove-plus-comm__3V2ASFe", + "r3-t900/prove-plus-comm__TBJs3k3", + "r3-t900/prove-plus-comm__w8dYms3", + "g1-t900/prove-plus-comm__X52UArL" + ] + }, + { + "id": 11, + "text": "Make the final check exit 0 only when the result is correct: print OK from inside the check (or chain with &&), never as a separate trailing line. In expect scripts a timed-out expect continues silently, so add a timeout branch that exits nonzero.", + "importance": 13, + "helpful": 13, + "harmful": 0, + "tasks": [ + "code-from-image", + "compile-compcert", + "crack-7z-hash", + "distribution-search", + "mteb-retrieve", + "openssl-selfsigned-cert", + "password-recovery", + "path-tracing", + "polyglot-rust-c", + "prove-plus-comm", + "qemu-alpine-ssh", + "query-optimize" + ], + "sources": [ + "pass1-a/prove-plus-comm__iSrWbGV", + "r1-t900/prove-plus-comm__W6wjt7b", + "r1-rep-t1800/crack-7z-hash__6WFusMK", + "r1-rep-t1800/mteb-retrieve__2j6H74W", + "r1-rep-t900/password-recovery__cYtKhXD", + "r1-rep-t900/prove-plus-comm__48FaXoU", + "r1-rep-t900/query-optimize__9jCUzwb", + "r3-t1200/code-from-image__Te2W2Q9", + "r3-t1200/code-from-image__gft5KDm", + "g1-t2400/compile-compcert__JEAGW63", + "r3-t3600/distribution-search__NkQ5dpe", + "r3-t3600/distribution-search__SpheBQt", + "g1-t900/openssl-selfsigned-cert__DqsGtus", + "r3-t1800/path-tracing__3P8c5eJ", + "r3-t1800/path-tracing__AZ2NaW7", + "r3-t900/polyglot-rust-c__7ZKNx9M", + "r3-t900/qemu-alpine-ssh__6o7vQiZ" + ] + }, + { + "id": 12, + "text": "When a build must carry a special property (coverage, soft-float, static linking), first grep the build's own option list for a built-in switch. Then don't trust a clean compile: exercise or inspect the binary (e.g. run it and confirm its runtime data files appear and parse).", + "importance": 3, + "helpful": 6, + "harmful": 0, + "tasks": [ + "make-doom-for-mips", + "sqlite-with-gcov" + ], + "sources": [ + "pass1-a/sqlite-with-gcov__rLEPiHR", + "r1-t900/sqlite-with-gcov__vPgwNLw", + "r1-rep-t900/sqlite-with-gcov__cTBWbG5", + "r1-rep-t900/sqlite-with-gcov__yrJJPSi", + "r3-t900/make-doom-for-mips__YxYGpeF", + "r3-t900/sqlite-with-gcov__JKx9qun", + "r3-t900/sqlite-with-gcov__QTZT6y5", + "r3-t900/sqlite-with-gcov__aTQSY3F", + "g1-t900/sqlite-with-gcov__C3fP2Et" + ] + }, + { + "id": 13, + "text": "To construct an object meeting a few numeric constraints over a huge space, use a small parametric family with as many continuous free parameters as constraints, solve with a root-finder, and assert the constraints within tolerance. If a discrete scan only gets close, add a continuous parameter.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "distribution-search" + ], + "sources": [ + "pass2-d/distribution-search__LUVHzza", + "r1-t3600/distribution-search__oum44Cc", + "r1-t3600/distribution-search__oum44Cc", + "r1-rep-t3600/distribution-search__k3ZP2tC", + "r1-rep-t3600/distribution-search__xyncuDG", + "r3-t3600/distribution-search__NkQ5dpe", + "r3-t3600/distribution-search__SpheBQt", + "r3-t3600/distribution-search__gg8S57L", + "g1-t3600/distribution-search__fukMp4t" + ] + }, + { + "id": 14, + "text": "After writing a result file, reload it in a separate fresh process and recompute every required property straight from the stated definitions, such as size, type, validity and target values, rather than trusting the variables that produced it.", + "importance": 13, + "helpful": 40, + "harmful": 0, + "tasks": [ + "code-from-image", + "crack-7z-hash", + "db-wal-recovery", + "distribution-search", + "dna-assembly", + "gcode-to-text", + "mteb-leaderboard", + "mteb-retrieve", + "multi-source-data-merger", + "openssl-selfsigned-cert", + "protein-assembly", + "pytorch-model-recovery" + ], + "sources": [ + "pass2-d/distribution-search__LUVHzza", + "pass2-c/dna-assembly__eAi9G5r", + "pass3-retry/pytorch-model-recovery__ienEJ35", + "pass4-2/code-from-image__nbysVEY", + "pass4-3/crack-7z-hash__BHu9tiH", + "pass4-1/multi-source-data-merger__ARn5yV3", + "pass4-3/protein-assembly__2VAycog", + "pass4-retry/mteb-leaderboard__99xLYSj", + "r1-t1200/code-from-image__efJoC2B", + "r1-t3600/distribution-search__oum44Cc", + "r1-t900/multi-source-data-merger__Y6TSdLB", + "r1-t1800/protein-assembly__bVCDQPN", + "r1-t900/pytorch-model-recovery__fUqzL2D", + "r1-rep-t1200/code-from-image__rMCvLne", + "r1-rep-t1200/code-from-image__w4JmQn5", + "r1-rep-t3600/distribution-search__k3ZP2tC", + "r1-rep-t3600/distribution-search__xyncuDG", + "r1-rep-t900/gcode-to-text__8nb77ua", + "r1-rep-t1800/mteb-retrieve__hH7GGyq", + "r1-rep-t900/multi-source-data-merger__JeRC3uo", + "r1-rep-t900/multi-source-data-merger__gaGXbjL", + "r1-rep-t900/pytorch-model-recovery__D27MtXA", + "r1-rep-t900/pytorch-model-recovery__rkLngbb", + "r3-t1200/code-from-image__DAaYNdw", + "r3-t1200/code-from-image__Te2W2Q9", + "r3-t1200/code-from-image__gft5KDm", + "g1-t1200/code-from-image__8NpVzYz", + "r3-t1800/crack-7z-hash__EnisgGH", + "g1-t900/db-wal-recovery__fWRTgbF", + "r3-t3600/distribution-search__NkQ5dpe", + "r3-t3600/distribution-search__SpheBQt", + "r3-t3600/distribution-search__gg8S57L", + "g1-t3600/distribution-search__fukMp4t", + "r3-t1800/mteb-retrieve__yPibexH", + "r3-t900/multi-source-data-merger__M9NwcKU", + "r3-t900/multi-source-data-merger__zWo3vRU", + "r3-t900/multi-source-data-merger__zq9qagK", + "g1-t900/multi-source-data-merger__jhsnfYB", + "g1-t900/openssl-selfsigned-cert__DqsGtus", + "r3-t900/pytorch-model-recovery__iaPeQHQ", + "g1-t900/pytorch-model-recovery__D2odrGY" + ] + }, + { + "id": 15, + "text": "When a requirement is ambiguous about exactly which part of your output is measured (which substring, which boundary, which convention), design so the constraint holds under every plausible interpretation instead of betting on one reading.", + "importance": 5, + "helpful": 12, + "harmful": 0, + "tasks": [ + "dna-assembly", + "openssl-selfsigned-cert", + "protein-assembly", + "pytorch-model-recovery" + ], + "sources": [ + "pass2-c/dna-assembly__eAi9G5r", + "pass3-retry/pytorch-model-recovery__ienEJ35", + "pass3-1/openssl-selfsigned-cert__68dv7GC", + "r1-t1800/dna-assembly__X4yEChU", + "r1-t900/openssl-selfsigned-cert__taa9Bhx", + "r1-rep-t900/openssl-selfsigned-cert__FP6muPF", + "r1-rep-t900/openssl-selfsigned-cert__cSf2h2w", + "r1-rep-t1800/protein-assembly__YpC4crr,r1-rep-t1800/protein-assembly__ZjZVrSf", + "r3-t1800/dna-assembly__uZczH9n,r3-t1800/dna-assembly__snHPgL4", + "r3-t900/openssl-selfsigned-cert__Jvn2v9p", + "r3-t900/openssl-selfsigned-cert__ixwYdd3", + "r3-t900/openssl-selfsigned-cert__qHzdJvH", + "g1-t900/pytorch-model-recovery__D2odrGY" + ] + }, + { + "id": 16, + "text": "If a value you must satisfy is normally computed by a standard domain tool, use that tool: search the repositories first, since it may be one install away. If it truly isn't available, implement the published formula, validate it against inputs with known reference values, and state the parameters you assumed.", + "importance": 5, + "helpful": 8, + "harmful": 0, + "tasks": [ + "crack-7z-hash", + "dna-assembly", + "mteb-leaderboard", + "sparql-university" + ], + "sources": [ + "pass2-c/dna-assembly__eAi9G5r", + "pass4-3/crack-7z-hash__BHu9tiH", + "r1-t1800/crack-7z-hash__FVP3vRh", + "r1-t1800/dna-assembly__X4yEChU", + "r1-t900/sparql-university__eeAPKyT", + "r1-rep-t1800/crack-7z-hash__6WFusMK", + "r1-rep-t1800/dna-assembly__2CYeXas", + "r1-rep-t1800/dna-assembly__XYwLaDE", + "r1-rep-t3600/mteb-leaderboard__WvUWzaR", + "g1-t1800/dna-assembly__N4iF44u" + ] + }, + { + "id": 17, + "text": "For filtering or sanitizing tasks, test both sides. Run a broad, known-adversarial corpus through the filter so every attack is neutralized. Also confirm benign input comes out unchanged apart from the preinstalled parser's own normalization, so over-stripping and additions get caught.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "filter-js-from-html" + ], + "sources": [ + "pass2-c/filter-js-from-html__HrunMyk", + "r3-t1800/filter-js-from-html__SqHW8Jw", + "g1-t1800/filter-js-from-html__5XRiT3p" + ] + }, + { + "id": 18, + "text": "When an external verification oracle (a browser, emulator or driver) fails to start after a quick diagnosis, switch to a cheaper check you can actually run, such as inspecting the outputs directly, instead of spending the time budget fixing the oracle, and state in your summary which check could not be done.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "filter-js-from-html" + ], + "sources": [ + "pass2-c/filter-js-from-html__HrunMyk" + ] + }, + { + "id": 19, + "text": "When a full build is slow or flaky, build only the smallest target the required acceptance check depends on rather than the default everything-plus-docs target, run it in the background with a log you poll, and protect enough of the time budget to actually run that check; blocking retry loops on the full build can use up the whole session.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "pass2-d/fix-ocaml-gc__gefTDzu" + ] + }, + { + "id": 20, + "text": "To read text hidden in geometric data (e.g. toolpaths), parse it into primitives, keep the relevant object via the file's own labels, and render an image. If it spans many layers or a tilted plane, project onto its best-fit plane (PCA), then flip or rotate until level and zoom on look-alikes.", + "importance": 2, + "helpful": 6, + "harmful": 0, + "tasks": [ + "gcode-to-text" + ], + "sources": [ + "pass2-a/gcode-to-text__ZedKas2", + "r1-t900/gcode-to-text__tyJktT9", + "r1-rep-t900/gcode-to-text__8nb77ua", + "r1-rep-t900/gcode-to-text__cKptXsz", + "r3-t900/gcode-to-text__C9BEpmi", + "r3-t900/gcode-to-text__maZQUAo", + "r3-t900/gcode-to-text__vESW7WK", + "g1-t900/gcode-to-text__3oWS95W" + ] + }, + { + "id": 21, + "text": "When drawing or processing tens of thousands of primitives, use one batched or vectorized call (a collection or array operation) rather than a per-item loop of library calls, and if a run stalls past a short wait, interrupt it and switch approach instead of waiting longer.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "gcode-to-text" + ], + "sources": [ + "pass2-a/gcode-to-text__ZedKas2", + "r1-t900/gcode-to-text__tyJktT9" + ] + }, + { + "id": 22, + "text": "When a task lists several required behaviours, exercise every one end-to-end the way a real caller would, even if a provided eval script passes, since it may cover only some. Have each step leave an observable effect (file, marker), check it, then revert temporary changes.", + "importance": 4, + "helpful": 13, + "harmful": 0, + "tasks": [ + "headless-terminal", + "mailman", + "openssl-selfsigned-cert" + ], + "sources": [ + "pass1-retry/headless-terminal__nYNzBHJ", + "r1-t900/headless-terminal__rA5DdVx", + "r1-t1800/mailman__ZyAANtB", + "r1-rep-t900/headless-terminal__BvcKNEk", + "r1-rep-t900/headless-terminal__kSkaTY5", + "r1-rep-t1800/mailman__BYPN2ip", + "r1-rep-t1800/mailman__zLGFpB6", + "r3-t900/headless-terminal__PSKUgve", + "r3-t900/headless-terminal__nGGGKt8", + "r3-t900/headless-terminal__vruErZs", + "g1-t900/headless-terminal__HQ3Jqaf", + "r3-t1800/mailman__5Txbugx,g1-t1800/mailman__6he4bwA", + "r3-t1800/mailman__MNiFx9c", + "r3-t1800/mailman__akb93wt", + "g1-t900/openssl-selfsigned-cert__DqsGtus" + ] + }, + { + "id": 23, + "text": "For constrained-edit puzzles, keep all chosen edits in one script that rebuilds the file from the pristine backup and runs the acceptance check. Loop over candidate edits (apply, measure, print a compact score), commit one improvement at a time, fixing the earliest failure first, then re-measure.", + "importance": 3, + "helpful": 5, + "harmful": 0, + "tasks": [ + "overfull-hbox", + "tune-mjcf" + ], + "sources": [ + "pass2-b/overfull-hbox__BbekRBh", + "pass2-retry-rosetta/tune-mjcf__6r8xAwD", + "r1-t750/overfull-hbox__ee9c4jM", + "r3-t750/overfull-hbox__4nUm7SC", + "r3-t750/overfull-hbox__YqMaoHV", + "r3-t750/overfull-hbox__nwPUQUF", + "g1-t750/overfull-hbox__qm66c8V" + ] + }, + { + "id": 24, + "text": "If a trial-and-revert loop is interrupted or times out, assume the working file may still hold a half-applied trial edit. Before finishing, diff the final file against the original backup at a fine granularity (for example word by word) and confirm that every change is one you intended and is allowed.", + "importance": 2, + "helpful": 6, + "harmful": 0, + "tasks": [ + "overfull-hbox" + ], + "sources": [ + "pass2-b/overfull-hbox__BbekRBh", + "r1-t750/overfull-hbox__ee9c4jM", + "r1-rep-t750/overfull-hbox__PX8tZXd", + "r1-rep-t750/overfull-hbox__kykPtUM", + "r3-t750/overfull-hbox__4nUm7SC", + "r3-t750/overfull-hbox__nwPUQUF", + "g1-t750/overfull-hbox__qm66c8V" + ] + }, + { + "id": 25, + "text": "Run the task's exact commands verbatim from a neutral directory and from the project folder, since paths resolve relative to the cwd. If your deliverable misses inputs from elsewhere, also look next to itself. A provided program that only works from its own folder is its convention, not your bug.", + "importance": 10, + "helpful": 14, + "harmful": 0, + "tasks": [ + "build-cython-ext", + "circuit-fibsqrt", + "gpt2-codegolf", + "make-mips-interpreter", + "openssl-selfsigned-cert", + "overfull-hbox", + "path-tracing", + "polyglot-c-py", + "polyglot-rust-c" + ], + "sources": [ + "pass2-a/polyglot-rust-c__JzYFKhY", + "pass3-1/gpt2-codegolf__kbBbdw9", + "r1-t900/build-cython-ext__eVWTonP", + "r1-t3600/circuit-fibsqrt__66WdFy2", + "r1-rep-t900/openssl-selfsigned-cert__FP6muPF", + "r1-rep-t900/polyglot-c-py__Sxig8bA", + "r1-rep-t900/polyglot-rust-c__ZtianBa", + "r3-t3600/circuit-fibsqrt__HiEsydF", + "r3-t3600/circuit-fibsqrt__tLnVfqd", + "r3-t900/gpt2-codegolf__MwZ5vUx", + "r3-t1800/make-mips-interpreter__RpYQ2XG", + "r3-t900/openssl-selfsigned-cert__ixwYdd3", + "r3-t900/openssl-selfsigned-cert__qHzdJvH", + "r3-t750/overfull-hbox__4nUm7SC", + "g1-t1800/path-tracing__hH2VFLW", + "r3-t900/polyglot-rust-c__Nr2bBxF", + "r3-t900/polyglot-rust-c__bPy8wEP" + ] + }, + { + "id": 26, + "text": "When a task asks for a numeric sequence or function of an unbounded input, check large inputs against both implementations and against values you already know, and use arbitrary-precision arithmetic rather than fixed-width integers, so silent overflow cannot produce wrong answers.", + "importance": 3, + "helpful": 9, + "harmful": 0, + "tasks": [ + "polyglot-c-py", + "polyglot-rust-c" + ], + "sources": [ + "pass2-a/polyglot-rust-c__JzYFKhY", + "pass3-1/polyglot-c-py__qTUX3L9", + "r1-t900/polyglot-c-py__tHhKQ5i", + "r1-t900/polyglot-rust-c__4jKjy4e", + "r1-rep-t900/polyglot-c-py__Sxig8bA", + "r1-rep-t900/polyglot-c-py__YVDj3cQ", + "r1-rep-t900/polyglot-rust-c__9qDL57L", + "r1-rep-t900/polyglot-rust-c__ZtianBa", + "r3-t900/polyglot-rust-c__7ZKNx9M", + "g1-t900/polyglot-rust-c__sQrHBkV" + ] + }, + { + "id": 27, + "text": "Order your endgame: finish edits and stop jobs you started, then run the final verification as its own plain command after the last write, not chained with cleanup, edits or guard tests (test -s, command -v ...). Prefer the task's stated check run verbatim and unpiped; it must write nothing.", + "importance": 32, + "helpful": 68, + "harmful": 0, + "tasks": [ + "build-pov-ray", + "circuit-fibsqrt", + "code-from-image", + "compile-compcert", + "crack-7z-hash", + "dna-assembly", + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis", + "fix-ocaml-gc", + "gcode-to-text", + "gpt2-codegolf", + "mailman", + "make-doom-for-mips", + "make-mips-interpreter", + "merge-diff-arc-agi-task", + "mteb-leaderboard", + "mteb-retrieve", + "multi-source-data-merger", + "overfull-hbox", + "password-recovery", + "path-tracing", + "polyglot-c-py", + "polyglot-rust-c", + "protein-assembly", + "prove-plus-comm", + "pytorch-model-cli", + "qemu-alpine-ssh", + "regex-log", + "sparql-university", + "sqlite-with-gcov", + "tune-mjcf" + ], + "sources": [ + "pass2-a/polyglot-rust-c__JzYFKhY", + "pass3-4/build-pov-ray__y3tJCzZ", + "pass3-3/circuit-fibsqrt__eLneuTd", + "pass2-retry-rosetta/tune-mjcf__6r8xAwD", + "pass4-2/code-from-image__nbysVEY", + "pass4-4/compile-compcert__JC6dk97", + "pass4-3/crack-7z-hash__BHu9tiH", + "pass4-1/password-recovery__WRY4VPe", + "pass4-3/protein-assembly__2VAycog", + "r1-t1200/code-from-image__efJoC2B", + "r1-t2400/compile-compcert__z8xLH5e", + "r1-t1800/crack-7z-hash__FVP3vRh", + "r1-t1800/mailman__ZyAANtB", + "r1-t3600/mteb-leaderboard__inv3Epe", + "r1-t1800/mteb-retrieve__3ptj2AR", + "r1-t900/password-recovery__4Y3Xr67", + "r1-t900/prove-plus-comm__W6wjt7b", + "r1-t900/sqlite-with-gcov__vPgwNLw", + "r1-rep-t12000/build-pov-ray__Q876nYw", + "r1-rep-t12000/build-pov-ray__sgwYSER", + "r1-rep-t1200/code-from-image__w4JmQn5", + "r1-rep-t2400/compile-compcert__UUx4DAT", + "r1-rep-t2400/compile-compcert__i9vxd3S", + "r1-rep-t1800/dna-assembly__XYwLaDE", + "r1-rep-t900/gcode-to-text__cKptXsz", + "r1-rep-t1800/mailman__zLGFpB6", + "r1-rep-t900/make-doom-for-mips__d3HJSmd,r1-rep-t900/make-doom-for-mips__vjS4U6V", + "r1-rep-t1800/make-mips-interpreter__HXABoiy", + "r1-rep-t3600/mteb-leaderboard__hCTcTFG", + "r1-rep-t1800/mteb-retrieve__2j6H74W", + "r1-rep-t1800/mteb-retrieve__hH7GGyq", + "r1-rep-t1800/path-tracing__E39pVqQ", + "r1-rep-t900/prove-plus-comm__48FaXoU", + "r1-rep-t900/prove-plus-comm__KpDxmYD", + "r1-rep-t900/qemu-alpine-ssh__Q2i7xiR", + "r3-t12000/build-pov-ray__Qo6eLfg", + "r3-t12000/build-pov-ray__WNR9rdw", + "r3-t12000/build-pov-ray__X6Sq7bk", + "r3-t1200/code-from-image__DAaYNdw", + "r3-t1800/crack-7z-hash__EnisgGH", + "r3-t1800/crack-7z-hash__GBpZR25", + "r3-t1800/crack-7z-hash__MbUQjgc", + "g1-t1800/dna-assembly__N4iF44u", + "r3-t1800/feal-differential-cryptanalysis__RcFHbqX", + "r3-t1800/feal-differential-cryptanalysis__noEe3qY", + "g1-t1800/feal-differential-cryptanalysis__4tyjr3w", + "g1-t1800/feal-linear-cryptanalysis__Vdtr4JC", + "r3-t3600/fix-ocaml-gc__DixbcMC", + "r3-t3600/fix-ocaml-gc__NLxdQwD", + "r3-t900/gcode-to-text__maZQUAo", + "r3-t900/gcode-to-text__vESW7WK", + "g1-t900/gcode-to-text__3oWS95W", + "r3-t900/gpt2-codegolf__uk4VPFE", + "r3-t900/make-doom-for-mips__6QzjHSh,g1-t900/make-doom-for-mips__NRPT2Mi", + "r3-t1800/make-mips-interpreter__gHkvYoE", + "r3-t900/merge-diff-arc-agi-task__jN5ZQCk", + "r3-t3600/mteb-leaderboard__4H2CZvZ", + "r3-t1800/mteb-retrieve__cVgJJGT", + "r3-t1800/mteb-retrieve__yPibexH", + "g1-t900/multi-source-data-merger__jhsnfYB", + "r3-t750/overfull-hbox__YqMaoHV", + "r3-t750/overfull-hbox__nwPUQUF", + "r3-t900/password-recovery__k354MXi", + "r3-t1800/path-tracing__3P8c5eJ", + "g1-t900/polyglot-c-py__zEW9c5y", + "r3-t900/polyglot-rust-c__Nr2bBxF", + "r3-t900/polyglot-rust-c__bPy8wEP", + "r3-t900/prove-plus-comm__3V2ASFe", + "r3-t900/prove-plus-comm__TBJs3k3", + "r3-t900/prove-plus-comm__w8dYms3", + "g1-t900/prove-plus-comm__X52UArL", + "r3-t900/pytorch-model-cli__iyVSYTp", + "r3-t900/qemu-alpine-ssh__vnBLusx", + "g1-t900/regex-log__TEgcQpd", + "r3-t900/sparql-university__QWQxg3J" + ] + }, + { + "id": 28, + "text": "When a file was saved by a heavy framework that is not installed, look inside it first, since many such formats are just archives of pickles and raw arrays. Often you can decode it with the standard library, for example a custom unpickler that maps the framework's rebuild hooks to plain readers, instead of spending the time budget installing the framework.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "pytorch-model-cli" + ], + "sources": [ + "pass2-a/pytorch-model-cli__8ikPT2P", + "r1-t900/pytorch-model-cli__SnGhQgT", + "r1-rep-t900/pytorch-model-cli__R67cSvk", + "r1-rep-t900/pytorch-model-cli__rkY9xfq", + "r3-t900/pytorch-model-cli__iyVSYTp", + "r3-t900/pytorch-model-cli__mB9QiSE", + "r3-t900/pytorch-model-cli__tiomLMm", + "g1-t900/pytorch-model-cli__qHP6PuT" + ] + }, + { + "id": 29, + "text": "When reimplementing a computation in another language, cross-check the result against an independent second implementation on the real input. Also run it under each plausible preprocessing convention (scaling, normalization, layout) to confirm the answer does not depend on an ambiguity you guessed at.", + "importance": 4, + "helpful": 4, + "harmful": 0, + "tasks": [ + "gpt2-codegolf", + "mteb-retrieve", + "pytorch-model-cli" + ], + "sources": [ + "pass2-a/pytorch-model-cli__8ikPT2P", + "r1-t1800/mteb-retrieve__3ptj2AR", + "r1-rep-t900/pytorch-model-cli__R67cSvk", + "g1-t900/gpt2-codegolf__fpNKH6U", + "g1-t900/pytorch-model-cli__qHP6PuT" + ] + }, + { + "id": 30, + "text": "Before committing to a long blocking wait on an interactive or slow process (a VM boot, an expect script, a build), take a quick short-timeout peek at its output to confirm it is progressing and that the channel you are watching actually carries its output, then poll in short intervals, so a silent stall cannot use up most of the time budget.", + "importance": 3, + "helpful": 1, + "harmful": 0, + "tasks": [ + "feal-linear-cryptanalysis", + "qemu-alpine-ssh" + ], + "sources": [ + "pass2-a/qemu-alpine-ssh__8uqjXHf", + "r3-t1800/feal-linear-cryptanalysis__ua3sLc7" + ] + }, + { + "id": 31, + "text": "When driving a guest system through a serial or other headless console, first confirm the guest is actually configured to print to that console (for example by capturing a screen image or checking the boot loader prompt) rather than waiting for a login prompt that may only appear on the graphical display.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "qemu-alpine-ssh" + ], + "sources": [ + "pass2-a/qemu-alpine-ssh__8uqjXHf" + ] + }, + { + "id": 32, + "text": "If a required library or runtime crashes on import or fails to start, spend the remaining time trying to get a working copy before falling back to guesswork, for example a different release or build installed into a separate virtual environment. Do not stop early and submit unverified changes to accuracy-sensitive settings while time is still left.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "tune-mjcf" + ], + "sources": [ + "pass2-a/tune-mjcf__eJYk53f" + ] + }, + { + "id": 33, + "text": "When compiling legacy C with a modern compiler, use permissive flags (e.g. -std=gnu89, -fcommon) and replace stale libc prototypes with standard headers. Recompile with -Wimplicit-function-declaration and -Wint-conversion to catch 64-bit pointer bugs, then compare -O0/-O2 outputs.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "build-pov-ray" + ], + "sources": [ + "pass3-4/build-pov-ray__y3tJCzZ", + "r1-t12000/build-pov-ray__Mu92oJj", + "r1-rep-t12000/build-pov-ray__Q876nYw", + "r1-rep-t12000/build-pov-ray__sgwYSER", + "r3-t12000/build-pov-ray__Qo6eLfg", + "r3-t12000/build-pov-ray__WNR9rdw", + "r3-t12000/build-pov-ray__X6Sq7bk", + "g1-t12000/build-pov-ray__tkNGeYU" + ] + }, + { + "id": 34, + "text": "When recovering a hidden secret (key, seed, parameter) from known input/output examples, accept a candidate only if it reproduces every known example. Then produce the final outputs with the task's own provided tool rather than a reimplementation, and spot-check that tool on a few known examples before writing the answer file.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "feal-linear-cryptanalysis" + ], + "sources": [ + "pass3-2/feal-linear-cryptanalysis__LNvJA8x", + "r1-t1800/feal-linear-cryptanalysis__LLRkaNf", + "r1-rep-t1800/feal-linear-cryptanalysis__8bCzGXs", + "r1-rep-t1800/feal-linear-cryptanalysis__nguJjR7", + "r3-t1800/feal-linear-cryptanalysis__LvKpnQG", + "r3-t1800/feal-linear-cryptanalysis__YJm4wj9", + "r3-t1800/feal-linear-cryptanalysis__ua3sLc7", + "g1-t1800/feal-linear-cryptanalysis__Vdtr4JC" + ] + }, + { + "id": 35, + "text": "When a search is too large to brute force jointly but splits into pieces linked by an equation, hash one piece and look up matches while enumerating the other (meet-in-the-middle), in compiled code. Use several independent relations as the match key so false matches stay rare.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "feal-linear-cryptanalysis" + ], + "sources": [ + "pass3-2/feal-linear-cryptanalysis__LNvJA8x", + "r1-rep-t1800/feal-linear-cryptanalysis__8bCzGXs", + "r1-rep-t1800/feal-linear-cryptanalysis__nguJjR7", + "r3-t1800/feal-linear-cryptanalysis__LvKpnQG", + "r3-t1800/feal-linear-cryptanalysis__ua3sLc7" + ] + }, + { + "id": 36, + "text": "When only a final linear stage may be adjusted on top of frozen components, compute the frozen features once and fit that stage in closed form (regularized least squares) rather than running an iterative training loop, then confirm by diffing every parameter against the original that only the allowed ones changed.", + "importance": 2, + "helpful": 6, + "harmful": 0, + "tasks": [ + "pytorch-model-recovery" + ], + "sources": [ + "pass3-retry/pytorch-model-recovery__ienEJ35", + "r1-t900/pytorch-model-recovery__fUqzL2D", + "r1-rep-t900/pytorch-model-recovery__D27MtXA", + "r1-rep-t900/pytorch-model-recovery__rkLngbb", + "r3-t900/pytorch-model-recovery__59kU9vX", + "r3-t900/pytorch-model-recovery__QPGAsmr", + "g1-t900/pytorch-model-recovery__D2odrGY" + ] + }, + { + "id": 37, + "text": "When testing code whose output is random, compare statistics to known truth with tolerances scaled to the standard error, then rerun under many seeds: a correct implementation's goodness-of-fit rejection rate should match the nominal level (about 5% at 0.05). A persistently higher rate suggests bias.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler" + ], + "sources": [ + "pass3-1/adaptive-rejection-sampler__vNJP6CU", + "r1-t900/adaptive-rejection-sampler__PArvFEg", + "r1-rep-t900/adaptive-rejection-sampler__HwnHbg3", + "r3-t900/adaptive-rejection-sampler__LrDGcCP", + "r3-t900/adaptive-rejection-sampler__j3gYRYy", + "r3-t900/adaptive-rejection-sampler__xezMuBL" + ] + }, + { + "id": 38, + "text": "Before declaring completion, stop background processes you started that the result does not need. Leave files in the task's directories in place; clear only your own scratch files in /tmp, and only if they could interfere. If ps/pgrep/pkill are missing, read /proc/*/cmdline to find, confirm or kill processes by command line.", + "importance": 10, + "helpful": 14, + "harmful": 0, + "tasks": [ + "gpt2-codegolf", + "headless-terminal", + "kv-store-grpc", + "make-mips-interpreter", + "merge-diff-arc-agi-task", + "polyglot-c-py", + "polyglot-rust-c", + "qemu-alpine-ssh", + "tune-mjcf" + ], + "sources": [ + "pass3-1/merge-diff-arc-agi-task__NtQSVZB", + "pass4-3/make-mips-interpreter__c8DW2ck", + "r1-t1800/make-mips-interpreter__W2F6ySm", + "r1-t900/polyglot-c-py__tHhKQ5i", + "r1-t900/polyglot-rust-c__4jKjy4e", + "r1-t900/tune-mjcf__mocj949", + "r1-rep-t900/gpt2-codegolf__mzFmpW3", + "r1-rep-t900/headless-terminal__BvcKNEk", + "r1-rep-t900/headless-terminal__kSkaTY5", + "r1-rep-t900/kv-store-grpc__7ynYVuN", + "r1-rep-t900/kv-store-grpc__H5mcipG", + "r1-rep-t900/merge-diff-arc-agi-task__SG6sUoA", + "r1-rep-t900/merge-diff-arc-agi-task__rrAgxvw", + "r1-rep-t900/polyglot-rust-c__9qDL57L", + "r1-rep-t900/qemu-alpine-ssh__Q2i7xiR", + "r3-t900/kv-store-grpc__jnvKq3w", + "r3-t900/merge-diff-arc-agi-task__LaYRW7y" + ] + }, + { + "id": 39, + "text": "Before writing a script a checker will run, test which third-party modules are installed; prefer the standard library with a fallback to an installed CLI tool. If you also use an optional package, rerun with its import blocked to prove the fallback path, and check the required output prints.", + "importance": 3, + "helpful": 7, + "harmful": 0, + "tasks": [ + "headless-terminal", + "openssl-selfsigned-cert" + ], + "sources": [ + "pass3-1/openssl-selfsigned-cert__68dv7GC", + "r1-t900/headless-terminal__rA5DdVx", + "r1-t900/openssl-selfsigned-cert__taa9Bhx", + "r1-rep-t900/openssl-selfsigned-cert__FP6muPF", + "r1-rep-t900/openssl-selfsigned-cert__cSf2h2w", + "r3-t900/headless-terminal__PSKUgve", + "r3-t900/openssl-selfsigned-cert__Jvn2v9p", + "r3-t900/openssl-selfsigned-cert__ixwYdd3", + "r3-t900/openssl-selfsigned-cert__qHzdJvH" + ] + }, + { + "id": 40, + "text": "If a runtime or tool needed to run or check the deliverable is missing, install it (the distro package for a library also pulls in its interpreter) and run every required invocation for real, rather than shipping something untested. Note any version mismatch in your summary.", + "importance": 6, + "helpful": 15, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler", + "crack-7z-hash", + "merge-diff-arc-agi-task", + "polyglot-c-py", + "sparql-university" + ], + "sources": [ + "pass3-1/polyglot-c-py__qTUX3L9", + "r1-t900/sparql-university__eeAPKyT", + "r1-rep-t900/adaptive-rejection-sampler__HwnHbg3", + "r1-rep-t900/adaptive-rejection-sampler__eh3usHZ", + "r1-rep-t900/merge-diff-arc-agi-task__SG6sUoA", + "r1-rep-t900/polyglot-c-py__Sxig8bA", + "r1-rep-t900/polyglot-c-py__YVDj3cQ", + "r1-rep-t900/sparql-university__jJ3NavW", + "r1-rep-t900/sparql-university__rihuQjY", + "r3-t900/adaptive-rejection-sampler__j3gYRYy", + "r3-t1800/crack-7z-hash__EnisgGH", + "r3-t1800/crack-7z-hash__GBpZR25", + "r3-t900/merge-diff-arc-agi-task__Eg7roT5", + "r3-t900/polyglot-c-py__PRGhezc,r3-t900/polyglot-c-py__nMBTWWS", + "g1-t900/polyglot-c-py__zEW9c5y", + "r3-t900/sparql-university__ZfKbYmZ", + "r3-t900/sparql-university__iUADnDm" + ] + }, + { + "id": 41, + "text": "When an optimization must reproduce the output of an original that is too slow to run, get the reference output by running the unchanged original in an accelerated scratch copy of the environment (for example a copied database with added indexes, leaving the provided original untouched), then compare your fast version's output byte for byte against it.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "query-optimize" + ], + "sources": [ + "pass3-1/query-optimize__E4BccaN", + "r3-t900/query-optimize__faCipRr", + "g1-t900/query-optimize__QqfAwP9" + ] + }, + { + "id": 42, + "text": "When transcribing logic or constants from an image or other lossy source, list the plausible readings of the ambiguous parts (look-alike characters, raw versus hex or encoded forms, lengths). Compute them all in one script and let any provided check value, such as a hinted prefix or a checksum, pick the right one, rather than committing to a single guess.", + "importance": 3, + "helpful": 9, + "harmful": 0, + "tasks": [ + "code-from-image", + "password-recovery" + ], + "sources": [ + "pass4-2/code-from-image__nbysVEY", + "r1-t1200/code-from-image__efJoC2B", + "r1-t900/password-recovery__4Y3Xr67", + "r1-rep-t1200/code-from-image__rMCvLne", + "r1-rep-t1200/code-from-image__w4JmQn5", + "r3-t1200/code-from-image__DAaYNdw", + "r3-t1200/code-from-image__Te2W2Q9", + "r3-t1200/code-from-image__gft5KDm", + "g1-t1200/code-from-image__8NpVzYz", + "r3-t900/password-recovery__k354MXi" + ] + }, + { + "id": 43, + "text": "After building a compiler, interpreter or similar tool, drive it through the whole pipeline on a small realistic sample (compile, link, run and check the output), because a successful build can still leave runtime libraries or headers missing from the locations the tool expects.", + "importance": 3, + "helpful": 8, + "harmful": 0, + "tasks": [ + "compile-compcert", + "make-doom-for-mips" + ], + "sources": [ + "pass4-4/compile-compcert__JC6dk97", + "r1-t2400/compile-compcert__z8xLH5e", + "r1-t900/make-doom-for-mips__BCT85A2", + "r1-rep-t2400/compile-compcert__UUx4DAT", + "r1-rep-t2400/compile-compcert__i9vxd3S", + "r3-t2400/compile-compcert__3FUpnGN", + "r3-t2400/compile-compcert__8y9sHD7", + "r3-t2400/compile-compcert__ho6QX8C", + "g1-t2400/compile-compcert__JEAGW63" + ] + }, + { + "id": 44, + "text": "When a file looks scrambled or encrypted, compare its first bytes with the format's documented magic header and look for long runs of one repeated byte (likely encoded zero padding); XOR often reveals a simple key. Confirm it by checking the decoded file passes the format's own integrity check.", + "importance": 2, + "helpful": 6, + "harmful": 0, + "tasks": [ + "db-wal-recovery" + ], + "sources": [ + "pass4-1/db-wal-recovery__jgiY4np", + "r1-t900/db-wal-recovery__qnwnRjY", + "r1-rep-t900/db-wal-recovery__AimMLx5", + "r1-rep-t900/db-wal-recovery__jkQmU3W", + "r3-t900/db-wal-recovery__GwatioF", + "r3-t900/db-wal-recovery__vPbhdHk", + "r3-t900/db-wal-recovery__xtS6cXe", + "g1-t900/db-wal-recovery__fWRTgbF" + ] + }, + { + "id": 45, + "text": "Opening a file with its normal library can silently change it (checkpointing, merging logs, deleting side files, creating locks), so do the inspection and extraction on a copy, then make sure the delivered files are in the intended state afterwards.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "db-wal-recovery" + ], + "sources": [ + "pass4-1/db-wal-recovery__jgiY4np", + "r1-t900/db-wal-recovery__qnwnRjY", + "r1-rep-t900/db-wal-recovery__AimMLx5", + "r1-rep-t900/db-wal-recovery__jkQmU3W", + "r3-t900/db-wal-recovery__GwatioF", + "r3-t900/db-wal-recovery__vPbhdHk", + "r3-t900/db-wal-recovery__xtS6cXe", + "g1-t900/db-wal-recovery__fWRTgbF" + ] + }, + { + "id": 46, + "text": "When code was changed but no history or diff exists, find the bug by comparing the suspect routine with sibling routines that walk the same data structure; a step, bound or invariant that differs from its siblings is usually the regression. Check every walker for that deviation and fix each minimally.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "pass2-retry-ocaml/fix-ocaml-gc__saSjV9F", + "r1-t3600/fix-ocaml-gc__uj5TJYh", + "r1-rep-t3600/fix-ocaml-gc__bkCHaww", + "r1-rep-t3600/fix-ocaml-gc__kwcsTRA", + "r3-t3600/fix-ocaml-gc__DixbcMC", + "r3-t3600/fix-ocaml-gc__LAzLaAE", + "r3-t3600/fix-ocaml-gc__NLxdQwD" + ] + }, + { + "id": 47, + "text": "When building an emulator, VM or interpreter, first disassemble the program and tally the instruction mnemonics to learn the exact subset used. Read any bundled source for how syscalls are numbered, since its wrappers may not follow the standard ABI. Implement that subset and fail loudly on anything else.", + "importance": 3, + "helpful": 7, + "harmful": 0, + "tasks": [ + "make-doom-for-mips", + "make-mips-interpreter" + ], + "sources": [ + "pass4-3/make-mips-interpreter__c8DW2ck", + "r1-t900/make-doom-for-mips__BCT85A2", + "r1-t1800/make-mips-interpreter__W2F6ySm", + "r1-rep-t900/make-doom-for-mips__d3HJSmd,r1-rep-t900/make-doom-for-mips__vjS4U6V", + "r1-rep-t1800/make-mips-interpreter__HXABoiy", + "r1-rep-t1800/make-mips-interpreter__NDNvtKA", + "r3-t900/make-doom-for-mips__KFVcsPX", + "r3-t1800/make-mips-interpreter__RpYQ2XG", + "r3-t1800/make-mips-interpreter__gHkvYoE", + "r3-t1800/make-mips-interpreter__nMGEqA8", + "g1-t1800/make-mips-interpreter__o9qjFks" + ] + }, + { + "id": 48, + "text": "When a specific package version is required and building from source would be slow, install prebuilt binaries from a repository snapshot pinned to a date when that version was current, so its dependencies are matching versions from the same time rather than the newest ones; then load the package and print the installed versions of it and its key dependencies to confirm.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "mcmc-sampling-stan" + ], + "sources": [ + "pass4-3/mcmc-sampling-stan__4SJPMbu", + "r3-t1800/mcmc-sampling-stan__LE2RGm3" + ] + }, + { + "id": 49, + "text": "When an optional library setting changes the result, let the task's wording decide: if it names roles like a query and documents, use the library's role-specific mode (e.g. a query prompt in the model's registered metadata), even if it needs a placeholder argument. Else use the default.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "mteb-retrieve" + ], + "sources": [ + "pass4-3/mteb-retrieve__cqBV9xL", + "r1-t1800/mteb-retrieve__3ptj2AR", + "r1-rep-t1800/mteb-retrieve__2j6H74W", + "r1-rep-t1800/mteb-retrieve__hH7GGyq", + "r3-t1800/mteb-retrieve__MtszDxs,g1-t1800/mteb-retrieve__NcPGgB6", + "r3-t1800/mteb-retrieve__MtszDxs,g1-t1800/mteb-retrieve__NcPGgB6", + "r3-t1800/mteb-retrieve__cVgJJGT", + "r3-t1800/mteb-retrieve__yPibexH" + ] + }, + { + "id": 50, + "text": "Before writing code that transforms or merges data, print every input's actual contents and column types in one exploratory command, so that schema gaps get noticed and handled on purpose, such as a required field missing from one source or an equivalent field stored in a different type or encoding. State each mapping choice you made in your summary.", + "importance": 3, + "helpful": 8, + "harmful": 0, + "tasks": [ + "multi-source-data-merger", + "raman-fitting" + ], + "sources": [ + "pass4-1/multi-source-data-merger__ARn5yV3", + "r1-t900/multi-source-data-merger__Y6TSdLB", + "r1-t900/raman-fitting__oN6bZVM", + "r1-rep-t900/multi-source-data-merger__JeRC3uo", + "r1-rep-t900/multi-source-data-merger__gaGXbjL", + "r3-t900/multi-source-data-merger__M9NwcKU", + "r3-t900/multi-source-data-merger__zWo3vRU", + "r3-t900/multi-source-data-merger__zq9qagK", + "g1-t900/multi-source-data-merger__jhsnfYB" + ] + }, + { + "id": 51, + "text": "Before editing a service's configuration, read its top-level config file and follow every include, because a provided environment may define its own server blocks or upstreams that override the distro defaults. If a request then returns a proxy error, check which backend the active config actually points at, and adapt to it rather than assuming your edited file is the one in effect.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "install-windows-3.11" + ], + "sources": [ + "pass4-5/install-windows-3.11__cD5fPWQ", + "r1-t3600/install-windows-3.11__osH4pRk", + "r1-rep-t3600/install-windows-3.11__ETFnasT", + "r1-rep-t3600/install-windows-3.11__eJS77YZ", + "r3-t3600/install-windows-3.11__4recKxm", + "r3-t3600/install-windows-3.11__Tr7Ftdg" + ] + }, + { + "id": 52, + "text": "When booting a provided disk image, read the guest's own config straight out of the image first (userspace filesystem tools at the partition offset, or raw byte search) to learn which drivers and hardware it expects, then pick matching emulated hardware instead of cycling options.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "install-windows-3.11" + ], + "sources": [ + "pass4-5/install-windows-3.11__cD5fPWQ", + "r1-rep-t3600/install-windows-3.11__ETFnasT", + "r3-t3600/install-windows-3.11__FWc6e4X", + "g1-t3600/install-windows-3.11__eo2TvaJ" + ] + }, + { + "id": 53, + "text": "When an answer depends on an eligibility filter (complete results, size limits, availability), print the top candidates both with and without the filter, together with the reason each one is excluded, so you can confirm the filter is what changes the winner and that no candidate was dropped by a loading or naming glitch.", + "importance": 2, + "helpful": 7, + "harmful": 0, + "tasks": [ + "mteb-leaderboard" + ], + "sources": [ + "pass4-retry/mteb-leaderboard__99xLYSj", + "r1-t3600/mteb-leaderboard__inv3Epe", + "r1-rep-t3600/mteb-leaderboard__WvUWzaR", + "r1-rep-t3600/mteb-leaderboard__hCTcTFG", + "r3-t3600/mteb-leaderboard__4H2CZvZ", + "r3-t3600/mteb-leaderboard__N7R2fzz", + "r3-t3600/mteb-leaderboard__rXGENDs", + "g1-t3600/mteb-leaderboard__ExALMnz" + ] + }, + { + "id": 54, + "text": "When writing numeric data files for others to read, use an explicit plain format (one value per line, fixed decimals) rather than default printing, then view the first lines and re-read the file to check for padding, scientific notation or mixed formats.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler" + ], + "sources": [ + "r1-t900/adaptive-rejection-sampler__PArvFEg", + "r1-rep-t900/adaptive-rejection-sampler__eh3usHZ" + ] + }, + { + "id": 55, + "text": "When porting to a new major library version, find every import alias and check each aliased attribute use against the installed library's real namespace (e.g. an AST walk with hasattr), which catches removed APIs a grep list misses. Pin the existing version via a constraints file.", + "importance": 2, + "helpful": 4, + "harmful": 0, + "tasks": [ + "build-cython-ext" + ], + "sources": [ + "r1-t900/build-cython-ext__eVWTonP", + "r1-rep-t900/build-cython-ext__2VMc5JK", + "r1-rep-t900/build-cython-ext__JSTJsaX", + "r3-t900/build-cython-ext__6K4WgKE", + "r3-t900/build-cython-ext__9FQm4A2", + "g1-t900/build-cython-ext__GyNHJk2" + ] + }, + { + "id": 56, + "text": "If the task invokes a program at a path where only its source exists, build it at exactly that path and run the stated commands verbatim there as the final check. Tests against a copy built elsewhere may not count as verifying the delivered state. Mention the extra built file in your summary.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "circuit-fibsqrt" + ], + "sources": [ + "r1-t3600/circuit-fibsqrt__66WdFy2", + "r3-t3600/circuit-fibsqrt__eP6oWWJ" + ] + }, + { + "id": 57, + "text": "When building old source with a newer packaged toolchain, try the build's own version-override option first. Then patch each identifier the newer toolchain removed to its documented replacement, one minimal edit per error, instead of building an exact older toolchain.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "compile-compcert" + ], + "sources": [ + "r1-t2400/compile-compcert__z8xLH5e", + "r1-rep-t2400/compile-compcert__UUx4DAT", + "r1-rep-t2400/compile-compcert__i9vxd3S", + "r3-t2400/compile-compcert__3FUpnGN", + "r3-t2400/compile-compcert__8y9sHD7", + "r3-t2400/compile-compcert__ho6QX8C" + ] + }, + { + "id": 58, + "text": "When your deliverable is an input to a downstream process you can model, write a simulator that reads only the delivered file plus the given inputs and reproduces the final target end-to-end; iterate until it matches exactly, since that catches orientation or ordering errors a checklist misses.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "dna-assembly" + ], + "sources": [ + "r1-t1800/dna-assembly__X4yEChU", + "r1-rep-t1800/dna-assembly__2CYeXas", + "r1-rep-t1800/dna-assembly__XYwLaDE", + "g1-t1800/dna-assembly__N4iF44u" + ] + }, + { + "id": 59, + "text": "When the task hints that a secret comes from a small seed or a reduced space, enumerate that reduced space and filter candidates with cheap consistency checks against chosen or observed data, instead of attacking the full space.", + "importance": 3, + "helpful": 6, + "harmful": 0, + "tasks": [ + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis" + ], + "sources": [ + "r1-t1800/feal-differential-cryptanalysis__Hr4kPkt", + "r1-t1800/feal-linear-cryptanalysis__LLRkaNf", + "r1-rep-t1800/feal-differential-cryptanalysis__XGgc5Fe", + "r1-rep-t1800/feal-differential-cryptanalysis__YQAZFGm", + "r3-t1800/feal-differential-cryptanalysis__RcFHbqX", + "r3-t1800/feal-differential-cryptanalysis__noEe3qY", + "r3-t1800/feal-linear-cryptanalysis__YJm4wj9" + ] + }, + { + "id": 60, + "text": "When several unknowns are jointly too large to search, look for relations or checks that involve only one unknown at a time (e.g. outermost layers), filter each piece independently first, then search the remaining pieces conditioned on the few survivors.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "feal-linear-cryptanalysis" + ], + "sources": [ + "r1-t1800/feal-linear-cryptanalysis__LLRkaNf", + "r1-rep-t1800/feal-linear-cryptanalysis__8bCzGXs", + "r1-rep-t1800/feal-linear-cryptanalysis__nguJjR7", + "r3-t1800/feal-linear-cryptanalysis__LvKpnQG", + "r3-t1800/feal-linear-cryptanalysis__YJm4wj9", + "g1-t1800/feal-linear-cryptanalysis__Vdtr4JC" + ] + }, + { + "id": 61, + "text": "When using a detector (browser, scanner, test harness) to prove that something harmful was removed, first confirm it flags each unfiltered positive control. Cases where the original never triggers prove nothing, so fix the trigger or verify those cases another way.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "filter-js-from-html" + ], + "sources": [ + "r1-t1800/filter-js-from-html__Q9y3YR3" + ] + }, + { + "id": 62, + "text": "After the required tests pass, also run the few neighbouring test groups that stress the subsystem you changed. A fix can pass the required suite and still break a related path, and these runs are cheap extra confidence.", + "importance": 2, + "helpful": 6, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "r1-t3600/fix-ocaml-gc__uj5TJYh", + "r1-rep-t3600/fix-ocaml-gc__bkCHaww", + "r1-rep-t3600/fix-ocaml-gc__kwcsTRA", + "r3-t3600/fix-ocaml-gc__DixbcMC", + "r3-t3600/fix-ocaml-gc__LAzLaAE", + "r3-t3600/fix-ocaml-gc__NLxdQwD", + "g1-t3600/fix-ocaml-gc__EXTWQgd" + ] + }, + { + "id": 63, + "text": "When no reference implementation can run, validate with a canonical input whose correct output is widely known, such as famous boilerplate text a model should complete, and also print intermediate values (token IDs, offsets) to compare against values you know are right.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "gpt2-codegolf" + ], + "sources": [ + "r1-t900/gpt2-codegolf__7kdzjfU", + "r1-rep-t900/gpt2-codegolf__iWbwUmN", + "r1-rep-t900/gpt2-codegolf__mzFmpW3", + "r3-t900/gpt2-codegolf__MwZ5vUx", + "r3-t900/gpt2-codegolf__uk4VPFE", + "r3-t900/gpt2-codegolf__ymxiZBG" + ] + }, + { + "id": 64, + "text": "When a binary data file has no index, first check that its size equals the expected element count times the element width, which shows it is raw and headerless. Then use per-segment statistics (scales near one, biases near zero) to infer storage order before writing the loader.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "gpt2-codegolf" + ], + "sources": [ + "r1-t900/gpt2-codegolf__7kdzjfU", + "r1-rep-t900/gpt2-codegolf__iWbwUmN", + "r3-t900/gpt2-codegolf__uk4VPFE", + "r3-t900/gpt2-codegolf__ymxiZBG", + "g1-t900/gpt2-codegolf__fpNKH6U" + ] + }, + { + "id": 65, + "text": "When a compiled program crashes or corrupts memory, rebuild it once with a memory sanitizer and debug info and rerun the same input. This points straight to the overflowing line and buffer, which beats guessing from the abort message.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "gpt2-codegolf" + ], + "sources": [ + "r1-t900/gpt2-codegolf__7kdzjfU" + ] + }, + { + "id": 66, + "text": "Read the full output of a passing smoke test for warnings or fallback notices, such as a program reporting a degraded mode caused by an inherited environment setting. Fix the cause in your deliverable rather than accepting the side effects of a test that only happened to pass.", + "importance": 3, + "helpful": 2, + "harmful": 0, + "tasks": [ + "headless-terminal", + "polyglot-c-py" + ], + "sources": [ + "r1-t900/headless-terminal__rA5DdVx", + "r1-rep-t900/polyglot-c-py__YVDj3cQ", + "r3-t900/polyglot-c-py__p8j9Tm8" + ] + }, + { + "id": 67, + "text": "When the task names a specific version known to work and the packaged newer version misbehaves, switch early to that exact version, building from source in the background with a minimal target list if needed, instead of spending the budget on workarounds for the newer one.", + "importance": 3, + "helpful": 7, + "harmful": 0, + "tasks": [ + "compile-compcert", + "install-windows-3.11" + ], + "sources": [ + "r1-t3600/install-windows-3.11__osH4pRk", + "r1-rep-t3600/install-windows-3.11__ETFnasT", + "r1-rep-t3600/install-windows-3.11__eJS77YZ", + "g1-t2400/compile-compcert__JEAGW63", + "r3-t3600/install-windows-3.11__4recKxm", + "r3-t3600/install-windows-3.11__FWc6e4X", + "r3-t3600/install-windows-3.11__Tr7Ftdg", + "g1-t3600/install-windows-3.11__eo2TvaJ" + ] + }, + { + "id": 68, + "text": "When a service runs as a dedicated unprivileged user, run its admin commands as that user rather than root. Afterwards, search its data, log and run directories for files with the wrong owner, because root-owned files left behind by your setup can quietly break the daemon later.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "mailman" + ], + "sources": [ + "r1-t1800/mailman__ZyAANtB", + "r1-rep-t1800/mailman__BYPN2ip", + "r3-t1800/mailman__MNiFx9c", + "r3-t1800/mailman__akb93wt" + ] + }, + { + "id": 69, + "text": "When no C compiler is installed, first start the distro package manager install (minimal, no recommends) in the background while you explore. If that is unavailable, try a pip-installable toolchain such as the ziglang package, which bundles clang plus libc headers.", + "importance": 4, + "helpful": 4, + "harmful": 0, + "tasks": [ + "make-doom-for-mips", + "pytorch-model-cli", + "sqlite-with-gcov" + ], + "sources": [ + "r1-t900/make-doom-for-mips__BCT85A2", + "r3-t900/make-doom-for-mips__6QzjHSh,g1-t900/make-doom-for-mips__NRPT2Mi", + "r3-t900/make-doom-for-mips__KFVcsPX", + "r3-t900/make-doom-for-mips__YxYGpeF", + "r3-t900/pytorch-model-cli__tiomLMm", + "r3-t900/sqlite-with-gcov__QTZT6y5" + ] + }, + { + "id": 70, + "text": "When the deliverable produces an image or other visual output, convert it to a format you can view and actually look at it. Check that the content is the expected scene, not just that the file exists with a valid header and size.", + "importance": 5, + "helpful": 9, + "harmful": 0, + "tasks": [ + "build-pov-ray", + "install-windows-3.11", + "make-doom-for-mips", + "make-mips-interpreter" + ], + "sources": [ + "r1-t1800/make-mips-interpreter__W2F6ySm", + "r1-rep-t1800/make-mips-interpreter__HXABoiy", + "r1-rep-t1800/make-mips-interpreter__NDNvtKA", + "r3-t12000/build-pov-ray__WNR9rdw", + "r3-t12000/build-pov-ray__X6Sq7bk", + "g1-t3600/install-windows-3.11__eo2TvaJ", + "r3-t900/make-doom-for-mips__KFVcsPX", + "r3-t900/make-doom-for-mips__YxYGpeF", + "r3-t1800/make-mips-interpreter__gHkvYoE", + "r3-t1800/make-mips-interpreter__nMGEqA8" + ] + }, + { + "id": 71, + "text": "When a language's package manager normally compiles packages from source on Linux (R being a common case), point it at a prebuilt-binary repository built for your exact OS release. This turns long compiles into quick downloads; afterwards, print the installed version to confirm it matches the requirement.", + "importance": 2, + "helpful": 4, + "harmful": 0, + "tasks": [ + "mcmc-sampling-stan" + ], + "sources": [ + "r1-t1800/mcmc-sampling-stan__YoaMuqy", + "r1-rep-t1800/mcmc-sampling-stan__6M43xXY", + "r1-rep-t1800/mcmc-sampling-stan__ogWSMmh", + "r3-t1800/mcmc-sampling-stan__2grZFD7", + "r3-t1800/mcmc-sampling-stan__bkVPVFd" + ] + }, + { + "id": 72, + "text": "For 'as of' a past date questions, check out the data's version-controlled source at the last commit before that date (shallow or sparse clone if large). Compute with the tool version released around then, reusing its own aggregation code instead of reimplementing it.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "mteb-leaderboard" + ], + "sources": [ + "r1-t3600/mteb-leaderboard__inv3Epe", + "r1-rep-t3600/mteb-leaderboard__WvUWzaR", + "r1-rep-t3600/mteb-leaderboard__hCTcTFG", + "r3-t3600/mteb-leaderboard__4H2CZvZ", + "r3-t3600/mteb-leaderboard__N7R2fzz", + "r3-t3600/mteb-leaderboard__rXGENDs", + "g1-t3600/mteb-leaderboard__ExALMnz" + ] + }, + { + "id": 73, + "text": "When recovering deleted data from a raw image, list byte offsets of each known piece separately (prefix, suffix, filename) plus the container's header signatures, since a full-pattern search misses split data. Hex-dump each hit, rejoin fragments, and accept only if a stored CRC or length matches.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "password-recovery" + ], + "sources": [ + "r1-t900/password-recovery__4Y3Xr67", + "r1-rep-t900/password-recovery__cYtKhXD", + "r1-rep-t900/password-recovery__ySvD2TH", + "r3-t900/password-recovery__RxgWyqW", + "r3-t900/password-recovery__Tz22qSr", + "r3-t900/password-recovery__k354MXi", + "g1-t900/password-recovery__Xj6JwVv" + ] + }, + { + "id": 74, + "text": "When inputs include a list of identifiers or reference files, treat it as a candidate menu: look up each candidate's properties in an authoritative source and pick by matching the task's stated numbers/criteria, rather than choosing from memory.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "protein-assembly" + ], + "sources": [ + "r1-t1800/protein-assembly__bVCDQPN" + ] + }, + { + "id": 75, + "text": "When a model or classifier's output cannot be checked against a reference runtime, inspect the input yourself, for example by rendering a small image as ASCII art with a quick standalone decoder, and confirm the prediction matches what a human would answer.", + "importance": 2, + "helpful": 4, + "harmful": 0, + "tasks": [ + "pytorch-model-cli" + ], + "sources": [ + "r1-t900/pytorch-model-cli__SnGhQgT", + "r1-rep-t900/pytorch-model-cli__rkY9xfq", + "r3-t900/pytorch-model-cli__iyVSYTp", + "r3-t900/pytorch-model-cli__mB9QiSE", + "r3-t900/pytorch-model-cli__tiomLMm" + ] + }, + { + "id": 76, + "text": "When a task forbids modifying an input file, record its checksum before any work and recompute it in your final verification, so you can prove the protected file stayed byte-identical.", + "importance": 3, + "helpful": 6, + "harmful": 0, + "tasks": [ + "pytorch-model-recovery", + "query-optimize" + ], + "sources": [ + "r1-t900/pytorch-model-recovery__fUqzL2D", + "r1-rep-t900/pytorch-model-recovery__D27MtXA", + "r1-rep-t900/pytorch-model-recovery__rkLngbb", + "r3-t900/pytorch-model-recovery__59kU9vX", + "r3-t900/pytorch-model-recovery__QPGAsmr", + "r3-t900/pytorch-model-recovery__iaPeQHQ", + "g1-t900/query-optimize__QqfAwP9" + ] + }, + { + "id": 77, + "text": "When rewriting code or a query to be faster but give identical output, first save the original's output as a reference. Query the data to confirm the invariants each simplification relies on (no orphan keys, uniqueness). Then byte-compare the new output, headers included.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "query-optimize" + ], + "sources": [ + "r1-t900/query-optimize__bmmaRnM", + "r1-rep-t900/query-optimize__9jCUzwb", + "r1-rep-t900/query-optimize__KtWTMbK", + "r3-t900/query-optimize__J7Lnk9f", + "r3-t900/query-optimize__TiunRY9", + "r3-t900/query-optimize__faCipRr" + ] + }, + { + "id": 78, + "text": "Before analysing measured data, check that its axis lands known reference features (textbook peak positions, expected ranges) where they belong. If not, try plausible unit conversions or reciprocals and the locale's number format (such as decimal commas) until the known landmarks line up, then state the conversion you used.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "raman-fitting" + ], + "sources": [ + "r1-t900/raman-fitting__oN6bZVM", + "r1-rep-t900/raman-fitting__3YhzPND", + "r1-rep-t900/raman-fitting__yJrVAGt", + "r3-t900/raman-fitting__gYiYG3u,r3-t900/raman-fitting__vPQbMrz", + "r3-t900/raman-fitting__gbKShkf", + "g1-t900/raman-fitting__AywfkbQ" + ] + }, + { + "id": 79, + "text": "When fitting a peak in a chosen data window, refit with narrower and wider windows. If parameters drift, use a wide window (many linewidths of baseline per side) where they level off, since narrow ones cut heavy tails and bias width/offset. Name the width/height convention.", + "importance": 2, + "helpful": 4, + "harmful": 0, + "tasks": [ + "raman-fitting" + ], + "sources": [ + "r1-t900/raman-fitting__oN6bZVM", + "r1-rep-t900/raman-fitting__3YhzPND", + "r1-rep-t900/raman-fitting__yJrVAGt", + "r3-t900/raman-fitting__gYiYG3u,r3-t900/raman-fitting__vPQbMrz", + "r3-t900/raman-fitting__gbKShkf", + "g1-t900/raman-fitting__AywfkbQ" + ] + }, + { + "id": 80, + "text": "When a hidden harness will load your deliverable (a pattern, config or script), mimic its exact consumption path: the same file read and strip, applied to the whole input with the same flags. If its runtime is absent, use a compatible engine and note it. Per-line or hand-pasted tests can disagree.", + "importance": 2, + "helpful": 4, + "harmful": 0, + "tasks": [ + "regex-log" + ], + "sources": [ + "r1-t900/regex-log__gT67tpE", + "r1-rep-t900/regex-log__reVXkR6", + "r3-t900/regex-log__TAnazuQ", + "r3-t900/regex-log__nzJvFZd", + "r3-t900/regex-log__xYSEQSM" + ] + }, + { + "id": 81, + "text": "When writing a query or filter over a small provided dataset, work out the expected result by hand first, including near-miss records and why each is excluded (missing end dates, typos, boundary dates). Then run the query in a real engine and confirm the output matches.", + "importance": 3, + "helpful": 7, + "harmful": 0, + "tasks": [ + "regex-log", + "sparql-university" + ], + "sources": [ + "r1-t900/sparql-university__eeAPKyT", + "r1-rep-t900/regex-log__F32Yejc", + "r1-rep-t900/sparql-university__jJ3NavW", + "r1-rep-t900/sparql-university__rihuQjY", + "r3-t900/sparql-university__QWQxg3J", + "r3-t900/sparql-university__ZfKbYmZ", + "r3-t900/sparql-university__iUADnDm", + "g1-t900/sparql-university__ie4Grns" + ] + }, + { + "id": 82, + "text": "To make a built tool available on PATH, link or install it into a standard bin directory that is already on the default PATH, rather than only editing a shell rc file. Then verify it resolves and runs from a fresh login shell started in an unrelated directory.", + "importance": 2, + "helpful": 6, + "harmful": 0, + "tasks": [ + "sqlite-with-gcov" + ], + "sources": [ + "r1-t900/sqlite-with-gcov__vPgwNLw", + "r1-rep-t900/sqlite-with-gcov__cTBWbG5", + "r1-rep-t900/sqlite-with-gcov__yrJJPSi", + "r3-t900/sqlite-with-gcov__JKx9qun", + "r3-t900/sqlite-with-gcov__QTZT6y5", + "r3-t900/sqlite-with-gcov__aTQSY3F", + "g1-t900/sqlite-with-gcov__C3fP2Et" + ] + }, + { + "id": 83, + "text": "When tuning for speed under an accuracy tolerance, write a small harness printing max error vs the reference and the time ratio over several seeds. Sweep candidate settings in one loop (solver choices, disabling unused features), then pick a winner with margin on both and recheck on many seeds.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "tune-mjcf" + ], + "sources": [ + "r1-t900/tune-mjcf__mocj949", + "r1-rep-t900/tune-mjcf__NzMyAPH", + "r1-rep-t900/tune-mjcf__VyEadw3", + "r3-t900/tune-mjcf__2i7JA4A", + "r3-t900/tune-mjcf__DbVdudc", + "r3-t900/tune-mjcf__ZbSzHAx", + "g1-t900/tune-mjcf__WZLDLoM" + ] + }, + { + "id": 84, + "text": "Before optimizing, use the tool's built-in profiler or timers to find which stage dominates the runtime, and also check which features are actually exercised (e.g. counts that stay zero). Then target the costly stage and switch off the idle features.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "tune-mjcf" + ], + "sources": [ + "r1-t900/tune-mjcf__mocj949", + "r1-rep-t900/tune-mjcf__NzMyAPH", + "r3-t900/tune-mjcf__2i7JA4A", + "r3-t900/tune-mjcf__ZbSzHAx" + ] + }, + { + "id": 85, + "text": "When installing a package with compiled extensions from source, also build them in place in the source tree. Then run the tests both inside the repo and on a copy outside it against the installed package, and compare compiled results with the pure-language fallback where one exists.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "build-cython-ext" + ], + "sources": [ + "r1-rep-t900/build-cython-ext__2VMc5JK", + "r1-rep-t900/build-cython-ext__JSTJsaX", + "r3-t900/build-cython-ext__6CmysLh", + "r3-t900/build-cython-ext__6K4WgKE" + ] + }, + { + "id": 86, + "text": "When a test fails inside code that consumes a third-party library's output (a missing key or attribute), probe the installed library with a tiny input and print the real structure, then patch the consumer to accept both old and new names rather than pinning or guessing.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "build-cython-ext" + ], + "sources": [ + "r1-rep-t900/build-cython-ext__JSTJsaX", + "r3-t900/build-cython-ext__6CmysLh", + "r3-t900/build-cython-ext__6K4WgKE", + "r3-t900/build-cython-ext__9FQm4A2" + ] + }, + { + "id": 87, + "text": "Check the exit status of every successful run of a program you built, not just its printed output. Legacy code with a void main or no final exit can return a garbage nonzero status that callers read as failure, so make it exit 0 explicitly and recheck.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "build-pov-ray" + ], + "sources": [ + "r1-rep-t12000/build-pov-ray__sgwYSER", + "g1-t12000/build-pov-ray__tkNGeYU" + ] + }, + { + "id": 88, + "text": "When the deliverable is a large machine-like artifact (a circuit, netlist, generated program or table), write a generator script built from small composable helpers and regenerate the artifact from it. A bug fix then becomes a one-line edit followed by a rerun of the full test sweep.", + "importance": 2, + "helpful": 5, + "harmful": 0, + "tasks": [ + "circuit-fibsqrt" + ], + "sources": [ + "r1-rep-t3600/circuit-fibsqrt__DNQpDEy", + "r1-rep-t3600/circuit-fibsqrt__DyUFemQ", + "r3-t3600/circuit-fibsqrt__HiEsydF", + "r3-t3600/circuit-fibsqrt__eP6oWWJ", + "r3-t3600/circuit-fibsqrt__tLnVfqd", + "g1-t3600/circuit-fibsqrt__vquinqL" + ] + }, + { + "id": 89, + "text": "When generating an artifact for a strict consumer, put assertions in the generator for the consumer's structural rules (ordering or index constraints, size limits, no dangling references). A violated assumption then fails loudly at generation time instead of yielding silently wrong output.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "circuit-fibsqrt" + ], + "sources": [ + "r1-rep-t3600/circuit-fibsqrt__DyUFemQ" + ] + }, + { + "id": 90, + "text": "When a search loop catches exceptions and then finds nothing, rerun a single representative case outside the try block with the full traceback shown. For root-finders, print the function's value at both bracket ends, because a hidden bad-bracket or domain error is often the real cause.", + "importance": 3, + "helpful": 1, + "harmful": 0, + "tasks": [ + "distribution-search", + "mteb-leaderboard" + ], + "sources": [ + "r1-rep-t3600/distribution-search__xyncuDG", + "r3-t3600/mteb-leaderboard__N7R2fzz" + ] + }, + { + "id": 91, + "text": "When both the inputs and the desired final result are given, first align or diff each input against the target to locate where every piece lands and what the junctions are; derive the design from that mapping instead of guessing, and keep the mapping script for the final end-to-end check.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "dna-assembly" + ], + "sources": [ + "r1-rep-t1800/dna-assembly__2CYeXas" + ] + }, + { + "id": 92, + "text": "When recovering an orphaned commit, create a named branch pointing at it before you merge or reset, so the work stays reachable even if the merge goes wrong or has to be aborted.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "fix-git" + ], + "sources": [ + "r1-rep-t900/fix-git__Fr6g7vD", + "g1-t900/fix-git__K4oagw6" + ] + }, + { + "id": 93, + "text": "When editing by line number (e.g. with sed), print the surrounding lines right after the edit to confirm the intended line changed and nothing nearby got clobbered, since line offsets are easy to get wrong.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "r1-rep-t3600/fix-ocaml-gc__bkCHaww" + ] + }, + { + "id": 94, + "text": "If you edit sources while a background build is running, rerun the build after it finishes so the final artifact includes every edit. For low-level fixes, also run a stress program under the project's own debug build with its built-in invariant checks turned on.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "r1-rep-t3600/fix-ocaml-gc__kwcsTRA" + ] + }, + { + "id": 95, + "text": "To prove a VM or GUI accepts programmatic input, capture the screen, send one harmless key or shortcut through the control channel, capture again and compare the captures. Then look at the image to confirm the expected reaction (e.g. a menu opened), and undo it.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "install-windows-3.11" + ], + "sources": [ + "r1-rep-t3600/install-windows-3.11__eJS77YZ" + ] + }, + { + "id": 96, + "text": "After running a code generator or build step, list the files it actually produced before you reference them in your code, because tools often rename outputs, for example by replacing characters that are illegal in identifiers. Then import or use the real names.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "kv-store-grpc" + ], + "sources": [ + "r1-rep-t900/kv-store-grpc__H5mcipG", + "r3-t900/kv-store-grpc__iLR9Ty8", + "r3-t900/kv-store-grpc__jnvKq3w" + ] + }, + { + "id": 97, + "text": "Apply a prescribed setting only to the scope the task names. Do not copy the same value onto analogous sibling settings without testing each one end-to-end, since it can silently disable another stated requirement (such as a confirmation step). Explain any deviation in your summary.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "mailman" + ], + "sources": [ + "r1-rep-t1800/mailman__zLGFpB6", + "r3-t1800/mailman__MNiFx9c", + "r3-t1800/mailman__akb93wt" + ] + }, + { + "id": 98, + "text": "For a long-running or never-exiting deliverable, measure how soon the required output appears. If a shorter run lacks it or it is incomplete, treat that as a defect to fix by cutting startup cost, not a quirk to explain away, since hidden checkers likely use modest timeouts.", + "importance": 3, + "helpful": 3, + "harmful": 0, + "tasks": [ + "make-doom-for-mips", + "make-mips-interpreter" + ], + "sources": [ + "r1-rep-t900/make-doom-for-mips__d3HJSmd,r1-rep-t900/make-doom-for-mips__vjS4U6V", + "r1-rep-t1800/make-mips-interpreter__NDNvtKA", + "r3-t1800/make-mips-interpreter__nMGEqA8", + "g1-t1800/make-mips-interpreter__o9qjFks" + ] + }, + { + "id": 99, + "text": "When an answer comes from MCMC or another stochastic estimator, cross-check it with an independent deterministic method, e.g. integrate out nuisance parameters analytically and grid-integrate the low-dimensional marginal (include the Jacobian for log-spaced grids). Agreement confirms model and code.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "mcmc-sampling-stan" + ], + "sources": [ + "r1-rep-t1800/mcmc-sampling-stan__6M43xXY", + "r3-t1800/mcmc-sampling-stan__LE2RGm3", + "r3-t1800/mcmc-sampling-stan__bkVPVFd", + "g1-t1800/mcmc-sampling-stan__P7QWU9f" + ] + }, + { + "id": 100, + "text": "When edits interact non-locally (one change reflows or shifts everything after it), stop greedy one-at-a-time fixes. Restrict to the affected region and, from a saved baseline, enumerate joint combinations of allowed choices (exhaustively if small, else randomly), ranking by the full check.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "overfull-hbox" + ], + "sources": [ + "r1-rep-t750/overfull-hbox__PX8tZXd", + "r1-rep-t750/overfull-hbox__kykPtUM" + ] + }, + { + "id": 101, + "text": "When a tool reports only failures, run a scratch copy with its diagnostics set to report everything (all lines and states, not only errors). That shows the context around the problem and which nearby choices actually affect it, while the delivered files stay unchanged.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "overfull-hbox" + ], + "sources": [ + "r1-rep-t750/overfull-hbox__PX8tZXd", + "r1-rep-t750/overfull-hbox__kykPtUM" + ] + }, + { + "id": 102, + "text": "When scripting languages and hex tools are missing, check which core tools remain before installing anything: perl is often present when python is not (its bundled zlib module computes CRC32), and grep -a -b -o for offsets plus dd and od for dumps cover most binary work.", + "importance": 3, + "helpful": 7, + "harmful": 0, + "tasks": [ + "password-recovery", + "regex-log" + ], + "sources": [ + "r1-rep-t900/password-recovery__cYtKhXD", + "r1-rep-t900/password-recovery__ySvD2TH", + "r1-rep-t900/regex-log__F32Yejc", + "r1-rep-t900/regex-log__reVXkR6", + "r3-t900/password-recovery__RxgWyqW", + "r3-t900/password-recovery__Tz22qSr", + "g1-t900/password-recovery__Xj6JwVv", + "r3-t900/regex-log__TAnazuQ", + "r3-t900/regex-log__nzJvFZd" + ] + }, + { + "id": 103, + "text": "When one source file must build and run under several toolchains or interpreters, compile it with every stated command verbatim and print the outputs side by side over a range of inputs, including edge cases and large values, so any divergence between the targets shows up at once.", + "importance": 3, + "helpful": 8, + "harmful": 0, + "tasks": [ + "polyglot-c-py", + "polyglot-rust-c" + ], + "sources": [ + "r1-rep-t900/polyglot-rust-c__9qDL57L", + "r1-rep-t900/polyglot-rust-c__ZtianBa", + "r3-t900/polyglot-c-py__PRGhezc,r3-t900/polyglot-c-py__nMBTWWS", + "r3-t900/polyglot-c-py__p8j9Tm8", + "g1-t900/polyglot-c-py__zEW9c5y", + "r3-t900/polyglot-rust-c__7ZKNx9M", + "r3-t900/polyglot-rust-c__Nr2bBxF", + "r3-t900/polyglot-rust-c__bPy8wEP", + "g1-t900/polyglot-rust-c__sQrHBkV" + ] + }, + { + "id": 104, + "text": "When building a deliverable out of records fetched from reference databases, copy each record verbatim and apply only the exact edit the task prescribes. Trimming extra leading/trailing bits you judged cosmetic can break a checker that compares your output against the untrimmed reference.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "protein-assembly" + ], + "sources": [ + "r1-rep-t1800/protein-assembly__YpC4crr,r1-rep-t1800/protein-assembly__ZjZVrSf", + "r3-t1800/protein-assembly__9rWMuWL,r3-t1800/protein-assembly__9fXvMs8" + ] + }, + { + "id": 105, + "text": "When optimizing a query or pipeline that ends in a top-N limit, compute the cheap ranking aggregate once, apply the limit first, and only then run the expensive per-row work (correlated subqueries, lookups) on the survivors, keeping the original tie-break order exactly.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "query-optimize" + ], + "sources": [ + "r1-rep-t900/query-optimize__KtWTMbK", + "r3-t900/query-optimize__J7Lnk9f", + "r3-t900/query-optimize__faCipRr" + ] + }, + { + "id": 106, + "text": "When the terminal may hide or truncate output lines, do not rely on reading test results by eye. Write the expected results to a file and diff the actual output against it, printing only OK or the mismatching lines, so every case is verified.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "regex-log" + ], + "sources": [ + "r1-rep-t900/regex-log__reVXkR6", + "r3-t900/regex-log__TAnazuQ", + "r3-t900/regex-log__xYSEQSM" + ] + }, + { + "id": 107, + "text": "When a task gives a reference date and uses words like 'current' or 'active', apply that date to every time-bounded field in the data (start and end dates). Decide explicitly how missing dates are treated, and say in your summary which records the date filter excluded.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "sparql-university" + ], + "sources": [ + "r1-rep-t900/sparql-university__rihuQjY", + "g1-t900/sparql-university__ie4Grns" + ] + }, + { + "id": 108, + "text": "When your code rejects bad inputs with a validity check, run it across many seeds and extreme but valid inputs (far-off locations, tiny scales) to catch false alarms. Rare spurious failures often come from values near underflow, so leave numerically unreliable points out of the check.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler" + ], + "sources": [ + "r3-t900/adaptive-rejection-sampler__LrDGcCP", + "g1-t900/adaptive-rejection-sampler__SPEQCFZ" + ] + }, + { + "id": 109, + "text": "Build the known-truth test suite to cover every code branch, such as bounded and unbounded domains or increasing and decreasing pieces. Then compare which cases pass and which fail: the property the failures share points at the branch with the bug, for example a sign error.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler" + ], + "sources": [ + "r3-t900/adaptive-rejection-sampler__xezMuBL" + ] + }, + { + "id": 110, + "text": "When a spec requires a check to fire at a specific phase (e.g. during processing, not just at setup), build a test input that passes the earlier phases and only violates later. Record that the violating path was actually reached, so the test proves the check works in that phase.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "adaptive-rejection-sampler" + ], + "sources": [ + "g1-t900/adaptive-rejection-sampler__SPEQCFZ" + ] + }, + { + "id": 111, + "text": "When building a source package whose compiled parts must match an already-installed dependency version, disable the installer's isolated build environment so it compiles against the installed version, not a fresh newest copy. Then print each extension's __file__ to rule out a silent pure-language fallback.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "build-cython-ext" + ], + "sources": [ + "r3-t900/build-cython-ext__6CmysLh", + "r3-t900/build-cython-ext__9FQm4A2", + "g1-t900/build-cython-ext__GyNHJk2" + ] + }, + { + "id": 112, + "text": "When a checker will compare your program's output to a reference, do not stop at a run that writes to a null sink. Write real output files, check their format and size and that the content is not trivial, and run the same input twice and byte-compare the results to confirm the output is deterministic.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "build-pov-ray" + ], + "sources": [ + "g1-t12000/build-pov-ray__tkNGeYU" + ] + }, + { + "id": 113, + "text": "When the real consumer is too slow for broad testing, write a fast independent emulator of it (e.g. bit-parallel, 64 cases per machine word). Test both ends of every piece of a piecewise target (floors, roots, thresholds), then confirm a sample on the real consumer.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "circuit-fibsqrt" + ], + "sources": [ + "g1-t3600/circuit-fibsqrt__vquinqL" + ] + }, + { + "id": 114, + "text": "After a numeric solver hits its target, rerun it on several other targets and sizes to expose fragile assumptions. If a bracketed root-finder then reports same-sign ends, step outward from the safe end to the first sign change instead of using extreme endpoints, where the function can turn around.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "distribution-search" + ], + "sources": [ + "g1-t3600/distribution-search__fukMp4t" + ] + }, + { + "id": 115, + "text": "When each item of a deliverable must follow a prescribed layout, build them all with the same canonical structure and no extra filler elements; ad-hoc padding added to tune one item can make a positional checker misread which part is which, even if your end-to-end simulation passes.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "dna-assembly" + ], + "sources": [ + "r3-t1800/dna-assembly__uZczH9n,r3-t1800/dna-assembly__snHPgL4" + ] + }, + { + "id": 116, + "text": "When you reimplement a provided primitive (for speed or inlining), assert it equals the original on many random inputs before building on it. If an attack or algorithm that should work succeeds only sometimes or yields no candidates, suspect a copied-helper bug such as missing masking before doubting the method.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "feal-differential-cryptanalysis" + ], + "sources": [ + "r3-t1800/feal-differential-cryptanalysis__FucyjUu" + ] + }, + { + "id": 117, + "text": "For a differential attack, trace the chosen input difference through the whitening, mixing and early rounds by hand. Look for one that passes deterministically (probability 1), so that every pair is a right pair and a single exact check can filter the last-round key candidates.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "feal-differential-cryptanalysis" + ], + "sources": [ + "r3-t1800/feal-differential-cryptanalysis__RcFHbqX", + "r3-t1800/feal-differential-cryptanalysis__noEe3qY", + "g1-t1800/feal-differential-cryptanalysis__4tyjr3w" + ] + }, + { + "id": 118, + "text": "When a harness calls your function, keep it free of side effects on shared state. For example, use a private RNG instance instead of the global one, and in your test driver assert that the caller's global state (RNG state, cwd, env) is unchanged after the call.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "feal-differential-cryptanalysis" + ], + "sources": [ + "g1-t1800/feal-differential-cryptanalysis__4tyjr3w" + ] + }, + { + "id": 119, + "text": "Treat preinstalled packages as hints about the hidden checker. A browser driver suggests results are judged by executing them; a preinstalled parser suggests which normalization is expected, so prefer it over downloading a different library whose output differs.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "filter-js-from-html" + ], + "sources": [ + "r3-t1800/filter-js-from-html__SqHW8Jw", + "r3-t1800/filter-js-from-html__WFc7rXA", + "g1-t1800/filter-js-from-html__5XRiT3p" + ] + }, + { + "id": 120, + "text": "When a task says output must match the input except for removing harmful parts, only delete. Never inject unrequested extras (new meta tags, added attributes, wrapper elements, hardening tweaks), and diff benign input against output to confirm nothing was added.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "filter-js-from-html" + ], + "sources": [ + "g1-t1800/filter-js-from-html__5XRiT3p" + ] + }, + { + "id": 121, + "text": "When looping a test or check command over several targets with filtered output, confirm each target printed its own summary. A missing one usually means the target name is wrong or nonexistent and nothing ran, so list the available targets and rerun the right ones.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "r3-t3600/fix-ocaml-gc__LAzLaAE" + ] + }, + { + "id": 122, + "text": "When a crash happens deep inside a build, get the exact failing command from the build tool's dry-run mode and rerun it under a debugger for a backtrace. After fixing, prove the fix with a regression test that crashes on a saved copy of the old binary and passes on the new one.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "fix-ocaml-gc" + ], + "sources": [ + "g1-t3600/fix-ocaml-gc__EXTWQgd" + ] + }, + { + "id": 123, + "text": "To view a plot without an imaging library, try a quick pip install first (often seconds). If that fails, write a minimal grayscale PNG with only the standard library (zlib plus struct). Split long, wide renders into overlapping tiles so each one stays readable.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "gcode-to-text" + ], + "sources": [ + "r3-t900/gcode-to-text__C9BEpmi", + "r3-t900/gcode-to-text__vESW7WK", + "g1-t900/gcode-to-text__3oWS95W" + ] + }, + { + "id": 124, + "text": "Compile with the task's exact stated command, keeping its argument order, from an unrelated directory. A library flag placed before the source file can fail to link. If you cannot change the command, change the code so it no longer depends on that flag's position.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "gpt2-codegolf" + ], + "sources": [ + "g1-t900/gpt2-codegolf__fpNKH6U" + ] + }, + { + "id": 125, + "text": "When testing a terminal, REPL or other interactive driver, have each typed command write its result to a scratch file and check those files, because screen output mixes echoed input, prompts and escape codes. Back up any user config you temporarily edit to test it, and restore it afterwards.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "headless-terminal" + ], + "sources": [ + "r3-t900/headless-terminal__PSKUgve", + "r3-t900/headless-terminal__nGGGKt8", + "r3-t900/headless-terminal__vruErZs", + "g1-t900/headless-terminal__HQ3Jqaf" + ] + }, + { + "id": 126, + "text": "When a deliverable spawns processes, threads or file descriptors, create and close it many times in a loop and assert that open-fd count, thread count and leftover child processes are unchanged. Include a child that ignores signals, so cleanup can't hang or leak.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "headless-terminal" + ], + "sources": [ + "g1-t900/headless-terminal__HQ3Jqaf" + ] + }, + { + "id": 127, + "text": "When unsure which settings fix a failure, launch every candidate configuration in parallel as separate scratch instances, each with its own log or control channel, then compare results together. If all fail identically, those settings aren't the cause, so move on, e.g. to the version named in the task.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "install-windows-3.11" + ], + "sources": [ + "r3-t3600/install-windows-3.11__4recKxm", + "r3-t3600/install-windows-3.11__Tr7Ftdg" + ] + }, + { + "id": 128, + "text": "When an emulator or tool misbehaves in a specific way (missing graphics, ignored ops), turn on its built-in trace events or guest-error logging, trigger one small action, and read the source path that handles the logged operations. A minimal targeted patch then rebuild often beats cycling options.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "install-windows-3.11" + ], + "sources": [ + "r3-t3600/install-windows-3.11__FWc6e4X" + ] + }, + { + "id": 129, + "text": "Configure a service through its stock, documented settings and change only what the task asks for. Avoid custom plugins, monkeypatches or overrides of built-in behaviour, and leave no extra files in the task directory. A hidden checker may drive the system in ways your additions change or break.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "mailman" + ], + "sources": [ + "r3-t1800/mailman__5Txbugx,g1-t1800/mailman__6he4bwA" + ] + }, + { + "id": 130, + "text": "Make the final check a fresh run of the user's exact command on the delivered artifact, judged by what that run itself produces. Assertions over logs, hashes or reports saved by earlier runs only show a past state and can hide a regression.", + "importance": 3, + "helpful": 1, + "harmful": 0, + "tasks": [ + "make-doom-for-mips", + "overfull-hbox" + ], + "sources": [ + "r3-t900/make-doom-for-mips__6QzjHSh,g1-t900/make-doom-for-mips__NRPT2Mi", + "r3-t750/overfull-hbox__YqMaoHV" + ] + }, + { + "id": 131, + "text": "When your deliverable produces an output file that a checker may poll for or read while the program keeps running, write it to a temporary file and rename it into place, so the file is never seen half-written.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "make-mips-interpreter" + ], + "sources": [ + "r3-t1800/make-mips-interpreter__RpYQ2XG" + ] + }, + { + "id": 132, + "text": "When a deliverable produces output (an image, a file) and no reference output exists, write a separate script that computes the expected result straight from the raw input assets and the source's own tables or formulas. Then compare byte for byte, rather than trusting a plausible-looking result.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "make-mips-interpreter" + ], + "sources": [ + "g1-t1800/make-mips-interpreter__o9qjFks" + ] + }, + { + "id": 133, + "text": "After an MCMC or other iterative estimator runs, read its diagnostics (divergences, R-hat, effective sample size). If it warns, rerun with stricter settings (e.g. a higher acceptance target) and check that the estimates stay stable before writing results.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "mcmc-sampling-stan" + ], + "sources": [ + "r3-t1800/mcmc-sampling-stan__2grZFD7", + "g1-t1800/mcmc-sampling-stan__P7QWU9f" + ] + }, + { + "id": 134, + "text": "After writing a model in a probabilistic language, evaluate its log density at many random parameter points, boundary values included, and compare with an independent hand-coded density. The difference must be the same constant at every point, which proves the model code is right before you trust any sampling.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "mcmc-sampling-stan" + ], + "sources": [ + "g1-t1800/mcmc-sampling-stan__P7QWU9f" + ] + }, + { + "id": 135, + "text": "Before fetching from a bundle or remote, list its refs and the commit each one points to, since HEAD and a named branch can point to different commits (one may be only the shared base). Fetch the ref that holds the actual work, then view the commit graph to confirm.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "merge-diff-arc-agi-task" + ], + "sources": [ + "r3-t900/merge-diff-arc-agi-task__Eg7roT5", + "r3-t900/merge-diff-arc-agi-task__LaYRW7y", + "r3-t900/merge-diff-arc-agi-task__jN5ZQCk", + "g1-t900/merge-diff-arc-agi-task__Wicye3a" + ] + }, + { + "id": 136, + "text": "When an installed library's loader always fetches the newest data, read its source to find the cache-directory setting and the no-update flag, then point it at your own pinned local checkout. That way the library's own parsing and aggregation run on the historical snapshot you chose.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "mteb-leaderboard" + ], + "sources": [ + "r3-t3600/mteb-leaderboard__rXGENDs" + ] + }, + { + "id": 137, + "text": "Don't trust a precomputed index file or an API listing to be complete. If a listing reports truncation, or results seem sparse, list everything cheaply with a shallow, blob-less clone (ls-tree) and fetch only the files you need. A full listing can reveal eligible candidates the index missed.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "mteb-leaderboard" + ], + "sources": [ + "g1-t3600/mteb-leaderboard__ExALMnz" + ] + }, + { + "id": 138, + "text": "For search-based fixing against a checker, score each trial by a graded value (number of violations weighted heavily, plus their total magnitude) rather than pass/fail. Try only edits near the reported violations, so the loop can accept partial progress and converges quickly.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "overfull-hbox" + ], + "sources": [ + "g1-t750/overfull-hbox__qm66c8V" + ] + }, + { + "id": 139, + "text": "Check the first easy hit (a grep match, an obvious file) against every property the task states, such as length, prefix, suffix and charset. If any one fails, treat it as a decoy or partial fragment and keep searching other offsets or copies rather than submitting it.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "password-recovery" + ], + "sources": [ + "r3-t900/password-recovery__RxgWyqW", + "r3-t900/password-recovery__Tz22qSr", + "g1-t900/password-recovery__Xj6JwVv" + ] + }, + { + "id": 140, + "text": "When a reimplementation's output nearly matches a reference, count the most common (yours, reference) value pairs among mismatches and print a coarse spatial map of where they differ. A dominant pair usually points straight at one wrong constant or branch, so it beats rereading the code.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "path-tracing" + ], + "sources": [ + "r3-t1800/path-tracing__AZ2NaW7" + ] + }, + { + "id": 141, + "text": "When reimplementing a program from its disassembly, mirror its exact numeric types and conversions: float versus double, casts around library calls like sqrt, and the order of operations. Aim for byte-identical output and check it with cmp, not just approximate similarity.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "path-tracing" + ], + "sources": [ + "r3-t1800/path-tracing__g32vhrn", + "g1-t1800/path-tracing__hH2VFLW" + ] + }, + { + "id": 142, + "text": "When the task hands you identifiers or accessions, pull each record from the database those IDs index and use its content verbatim (expanding only placeholders it documents). A second database's entry for the 'same' thing can be a trimmed or variant copy that a checker won't match.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "protein-assembly" + ], + "sources": [ + "r3-t1800/protein-assembly__9rWMuWL,r3-t1800/protein-assembly__9fXvMs8" + ] + }, + { + "id": 143, + "text": "Run the final verification as a command whose own text shows the check (a named script or a one-liner, not a heredoc piped into an interpreter). Give it a long enough wait to finish in that same call, since a result collected later by polling may not count as verifying your last change.", + "importance": 6, + "helpful": 4, + "harmful": 0, + "tasks": [ + "protein-assembly", + "prove-plus-comm", + "qemu-alpine-ssh", + "raman-fitting", + "sparql-university" + ], + "sources": [ + "r3-t1800/protein-assembly__9rWMuWL,r3-t1800/protein-assembly__9fXvMs8", + "r3-t900/prove-plus-comm__TBJs3k3", + "g1-t900/prove-plus-comm__X52UArL", + "g1-t900/qemu-alpine-ssh__qk4qmSY", + "g1-t900/raman-fitting__AywfkbQ", + "g1-t900/sparql-university__ie4Grns" + ] + }, + { + "id": 144, + "text": "When rebuilding an architecture from saved parameters, load with strict key and shape matching and regenerate deterministic buffers, comparing them to the stored ones within float tolerance. Sweep hyperparameters that leave no trace in shapes (e.g. head count) and report your choice.", + "importance": 2, + "helpful": 1, + "harmful": 0, + "tasks": [ + "pytorch-model-recovery" + ], + "sources": [ + "r3-t900/pytorch-model-recovery__59kU9vX", + "r3-t900/pytorch-model-recovery__QPGAsmr", + "r3-t900/pytorch-model-recovery__iaPeQHQ" + ] + }, + { + "id": 145, + "text": "To replace a correlated 'top item per group' subquery (order by count desc, id asc, limit 1), aggregate once per group and pack the sort key and the tie-break into one integer for MAX (e.g. count*BIG - id), then decode both parts arithmetically. Check that BIG exceeds the id range.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "query-optimize" + ], + "sources": [ + "r3-t900/query-optimize__TiunRY9" + ] + }, + { + "id": 146, + "text": "To prove a rewritten query or transform is equivalent, also diff it against the original on small synthetic fixtures that break the real data's incidental invariants (duplicate keys, ties, orphan references, empty tables), randomized over many seeds. Equivalence must hold for any data, not just this data.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "query-optimize" + ], + "sources": [ + "g1-t900/query-optimize__QqfAwP9" + ] + }, + { + "id": 147, + "text": "For regexes consumed by findall, remember that capture groups change what is returned: use exactly one group for the wanted part and non-capturing groups elsewhere. Put line-wide conditions in a lookahead anchored at line start, and use a greedy prefix to pick the last occurrence.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "regex-log" + ], + "sources": [ + "r3-t900/regex-log__nzJvFZd", + "r3-t900/regex-log__xYSEQSM", + "g1-t900/regex-log__TEgcQpd" + ] + }, + { + "id": 148, + "text": "When a randomized comparison against your own reference reports a mismatch, first check the failing case by hand against the spec, since the reference is often the buggy side. Fix whichever is wrong and rerun the full sweep, rather than reflexively bending the deliverable to match the reference.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "regex-log" + ], + "sources": [ + "g1-t900/regex-log__TEgcQpd" + ] + }, + { + "id": 149, + "text": "If the package manager says it cannot locate a package in a fresh container, the package index is probably empty. Run the index update before the install, in the background with a log, rather than giving up on the tool or switching to manual workarounds.", + "importance": 2, + "helpful": 2, + "harmful": 0, + "tasks": [ + "sparql-university" + ], + "sources": [ + "r3-t900/sparql-university__QWQxg3J", + "r3-t900/sparql-university__ZfKbYmZ", + "r3-t900/sparql-university__iUADnDm" + ] + }, + { + "id": 150, + "text": "When a faster algorithm or solver nearly meets an accuracy tolerance, tighten its own convergence tolerance or iteration settings and re-measure before discarding it. Extra precision is often cheap and buys a safe accuracy margin while keeping most of the speedup.", + "importance": 2, + "helpful": 3, + "harmful": 0, + "tasks": [ + "tune-mjcf" + ], + "sources": [ + "r3-t900/tune-mjcf__2i7JA4A", + "r3-t900/tune-mjcf__DbVdudc", + "r3-t900/tune-mjcf__ZbSzHAx", + "g1-t900/tune-mjcf__WZLDLoM" + ] + }, + { + "id": 151, + "text": "When only some settings of a config or model file may change, parse both the original and your edited version, remove the intended edits, and assert the remaining trees are identical. Also compare the compiled or loaded values of the protected properties, which proves nothing forbidden changed.", + "importance": 2, + "helpful": 0, + "harmful": 0, + "tasks": [ + "tune-mjcf" + ], + "sources": [ + "g1-t900/tune-mjcf__WZLDLoM" + ] + } + ], + "learned": [ + "g1-t1200/code-from-image__8NpVzYz", + "g1-t12000/build-pov-ray__tkNGeYU", + "g1-t1800/dna-assembly__N4iF44u", + "g1-t1800/feal-differential-cryptanalysis__4tyjr3w", + "g1-t1800/feal-linear-cryptanalysis__Vdtr4JC", + "g1-t1800/filter-js-from-html__5XRiT3p", + "g1-t1800/mailman__6he4bwA", + "g1-t1800/make-mips-interpreter__o9qjFks", + "g1-t1800/mcmc-sampling-stan__P7QWU9f", + "g1-t1800/mteb-retrieve__NcPGgB6", + "g1-t1800/path-tracing__hH2VFLW", + "g1-t2400/compile-compcert__JEAGW63", + "g1-t3600/circuit-fibsqrt__vquinqL", + "g1-t3600/distribution-search__fukMp4t", + "g1-t3600/fix-ocaml-gc__EXTWQgd", + "g1-t3600/install-windows-3.11__eo2TvaJ", + "g1-t3600/mteb-leaderboard__ExALMnz", + "g1-t750/overfull-hbox__qm66c8V", + "g1-t900/adaptive-rejection-sampler__SPEQCFZ", + "g1-t900/build-cython-ext__GyNHJk2", + "g1-t900/db-wal-recovery__fWRTgbF", + "g1-t900/fix-git__K4oagw6", + "g1-t900/gcode-to-text__3oWS95W", + "g1-t900/gpt2-codegolf__fpNKH6U", + "g1-t900/headless-terminal__HQ3Jqaf", + "g1-t900/kv-store-grpc__mf7amCP", + "g1-t900/make-doom-for-mips__NRPT2Mi", + "g1-t900/merge-diff-arc-agi-task__Wicye3a", + "g1-t900/multi-source-data-merger__jhsnfYB", + "g1-t900/openssl-selfsigned-cert__DqsGtus", + "g1-t900/password-recovery__Xj6JwVv", + "g1-t900/polyglot-c-py__zEW9c5y", + "g1-t900/polyglot-rust-c__sQrHBkV", + "g1-t900/prove-plus-comm__X52UArL", + "g1-t900/pytorch-model-cli__qHP6PuT", + "g1-t900/pytorch-model-recovery__D2odrGY", + "g1-t900/qemu-alpine-ssh__qk4qmSY", + "g1-t900/query-optimize__QqfAwP9", + "g1-t900/raman-fitting__AywfkbQ", + "g1-t900/regex-log__TEgcQpd", + "g1-t900/sparql-university__ie4Grns", + "g1-t900/sqlite-with-gcov__C3fP2Et", + "g1-t900/tune-mjcf__WZLDLoM", + "m1-fix-git/fix-git__rznNHc5", + "m2-fix-git/fix-git__oGtXbYW", + "m3-fix-git/fix-git__D3yv22h", + "m4-fix-git/fix-git__h7ah2xR", + "pass1-a/build-cython-ext__LsXpBxY", + "pass1-a/make-doom-for-mips__xF5WLFV", + "pass1-a/prove-plus-comm__iSrWbGV", + "pass1-a/raman-fitting__RT97YZb", + "pass1-a/regex-log__mzzwzMv", + "pass1-a/sqlite-with-gcov__rLEPiHR", + "pass1-b/feal-differential-cryptanalysis__6H9BX99", + "pass1-b/mailman__w5aZRRZ", + "pass1-b/path-tracing__wSyxsuh", + "pass1-retry/headless-terminal__nYNzBHJ", + "pass2-a/gcode-to-text__ZedKas2", + "pass2-a/polyglot-rust-c__JzYFKhY", + "pass2-a/pytorch-model-cli__8ikPT2P", + "pass2-a/qemu-alpine-ssh__8uqjXHf", + "pass2-a/tune-mjcf__eJYk53f", + "pass2-b/overfull-hbox__BbekRBh", + "pass2-c/dna-assembly__eAi9G5r", + "pass2-c/filter-js-from-html__HrunMyk", + "pass2-d/distribution-search__LUVHzza", + "pass2-d/fix-ocaml-gc__gefTDzu", + "pass2-retry-ocaml/fix-ocaml-gc__saSjV9F", + "pass2-retry-rosetta/tune-mjcf__6r8xAwD", + "pass3-1/adaptive-rejection-sampler__vNJP6CU", + "pass3-1/gpt2-codegolf__kbBbdw9", + "pass3-1/merge-diff-arc-agi-task__NtQSVZB", + "pass3-1/openssl-selfsigned-cert__68dv7GC", + "pass3-1/polyglot-c-py__qTUX3L9", + "pass3-1/query-optimize__E4BccaN", + "pass3-2/feal-linear-cryptanalysis__LNvJA8x", + "pass3-3/circuit-fibsqrt__eLneuTd", + "pass3-4/build-pov-ray__y3tJCzZ", + "pass3-retry/pytorch-model-recovery__ienEJ35", + "pass4-1/db-wal-recovery__jgiY4np", + "pass4-1/multi-source-data-merger__ARn5yV3", + "pass4-1/password-recovery__WRY4VPe", + "pass4-1/sparql-university__8LXFEeN", + "pass4-2/code-from-image__nbysVEY", + "pass4-3/crack-7z-hash__BHu9tiH", + "pass4-3/make-mips-interpreter__c8DW2ck", + "pass4-3/mcmc-sampling-stan__4SJPMbu", + "pass4-3/mteb-retrieve__cqBV9xL", + "pass4-3/protein-assembly__2VAycog", + "pass4-4/compile-compcert__JC6dk97", + "pass4-5/install-windows-3.11__cD5fPWQ", + "pass4-retry/mteb-leaderboard__99xLYSj", + "r1-qemu/qemu-alpine-ssh__koyscAj", + "r1-rep-t1200/code-from-image__rMCvLne", + "r1-rep-t1200/code-from-image__w4JmQn5", + "r1-rep-t12000/build-pov-ray__Q876nYw", + "r1-rep-t12000/build-pov-ray__sgwYSER", + "r1-rep-t1800/crack-7z-hash__6WFusMK", + "r1-rep-t1800/dna-assembly__2CYeXas", + "r1-rep-t1800/dna-assembly__XYwLaDE", + "r1-rep-t1800/feal-differential-cryptanalysis__XGgc5Fe", + "r1-rep-t1800/feal-differential-cryptanalysis__YQAZFGm", + "r1-rep-t1800/feal-linear-cryptanalysis__8bCzGXs", + "r1-rep-t1800/feal-linear-cryptanalysis__nguJjR7", + "r1-rep-t1800/filter-js-from-html__9Gvjm6N", + "r1-rep-t1800/filter-js-from-html__C6Gpo6Y", + "r1-rep-t1800/mailman__BYPN2ip", + "r1-rep-t1800/mailman__zLGFpB6", + "r1-rep-t1800/make-mips-interpreter__HXABoiy", + "r1-rep-t1800/make-mips-interpreter__NDNvtKA", + "r1-rep-t1800/mcmc-sampling-stan__6M43xXY", + "r1-rep-t1800/mcmc-sampling-stan__ogWSMmh", + "r1-rep-t1800/mteb-retrieve__2j6H74W", + "r1-rep-t1800/mteb-retrieve__hH7GGyq", + "r1-rep-t1800/path-tracing__E39pVqQ", + "r1-rep-t1800/path-tracing__GugRcxU", + "r1-rep-t1800/protein-assembly__YpC4crr", + "r1-rep-t1800/protein-assembly__ZjZVrSf", + "r1-rep-t2400/compile-compcert__UUx4DAT", + "r1-rep-t2400/compile-compcert__i9vxd3S", + "r1-rep-t3600/circuit-fibsqrt__DNQpDEy", + "r1-rep-t3600/circuit-fibsqrt__DyUFemQ", + "r1-rep-t3600/distribution-search__k3ZP2tC", + "r1-rep-t3600/distribution-search__xyncuDG", + "r1-rep-t3600/fix-ocaml-gc__bkCHaww", + "r1-rep-t3600/fix-ocaml-gc__kwcsTRA", + "r1-rep-t3600/install-windows-3.11__ETFnasT", + "r1-rep-t3600/install-windows-3.11__eJS77YZ", + "r1-rep-t3600/mteb-leaderboard__WvUWzaR", + "r1-rep-t3600/mteb-leaderboard__hCTcTFG", + "r1-rep-t750/overfull-hbox__PX8tZXd", + "r1-rep-t750/overfull-hbox__kykPtUM", + "r1-rep-t900/adaptive-rejection-sampler__HwnHbg3", + "r1-rep-t900/adaptive-rejection-sampler__eh3usHZ", + "r1-rep-t900/build-cython-ext__2VMc5JK", + "r1-rep-t900/build-cython-ext__JSTJsaX", + "r1-rep-t900/db-wal-recovery__AimMLx5", + "r1-rep-t900/db-wal-recovery__jkQmU3W", + "r1-rep-t900/fix-git__EWnZRi7", + "r1-rep-t900/fix-git__Fr6g7vD", + "r1-rep-t900/gcode-to-text__8nb77ua", + "r1-rep-t900/gcode-to-text__cKptXsz", + "r1-rep-t900/gpt2-codegolf__iWbwUmN", + "r1-rep-t900/gpt2-codegolf__mzFmpW3", + "r1-rep-t900/headless-terminal__BvcKNEk", + "r1-rep-t900/headless-terminal__kSkaTY5", + "r1-rep-t900/kv-store-grpc__7ynYVuN", + "r1-rep-t900/kv-store-grpc__H5mcipG", + "r1-rep-t900/make-doom-for-mips__d3HJSmd", + "r1-rep-t900/make-doom-for-mips__vjS4U6V", + "r1-rep-t900/merge-diff-arc-agi-task__SG6sUoA", + "r1-rep-t900/merge-diff-arc-agi-task__rrAgxvw", + "r1-rep-t900/multi-source-data-merger__JeRC3uo", + "r1-rep-t900/multi-source-data-merger__gaGXbjL", + "r1-rep-t900/openssl-selfsigned-cert__FP6muPF", + "r1-rep-t900/openssl-selfsigned-cert__cSf2h2w", + "r1-rep-t900/password-recovery__cYtKhXD", + "r1-rep-t900/password-recovery__ySvD2TH", + "r1-rep-t900/polyglot-c-py__Sxig8bA", + "r1-rep-t900/polyglot-c-py__YVDj3cQ", + "r1-rep-t900/polyglot-rust-c__9qDL57L", + "r1-rep-t900/polyglot-rust-c__ZtianBa", + "r1-rep-t900/prove-plus-comm__48FaXoU", + "r1-rep-t900/prove-plus-comm__KpDxmYD", + "r1-rep-t900/pytorch-model-cli__R67cSvk", + "r1-rep-t900/pytorch-model-cli__rkY9xfq", + "r1-rep-t900/pytorch-model-recovery__D27MtXA", + "r1-rep-t900/pytorch-model-recovery__rkLngbb", + "r1-rep-t900/qemu-alpine-ssh__Q2i7xiR", + "r1-rep-t900/qemu-alpine-ssh__jLsUsZ6", + "r1-rep-t900/query-optimize__9jCUzwb", + "r1-rep-t900/query-optimize__KtWTMbK", + "r1-rep-t900/raman-fitting__3YhzPND", + "r1-rep-t900/raman-fitting__yJrVAGt", + "r1-rep-t900/regex-log__F32Yejc", + "r1-rep-t900/regex-log__reVXkR6", + "r1-rep-t900/sparql-university__jJ3NavW", + "r1-rep-t900/sparql-university__rihuQjY", + "r1-rep-t900/sqlite-with-gcov__cTBWbG5", + "r1-rep-t900/sqlite-with-gcov__yrJJPSi", + "r1-rep-t900/tune-mjcf__NzMyAPH", + "r1-rep-t900/tune-mjcf__VyEadw3", + "r1-t1200/code-from-image__efJoC2B", + "r1-t12000/build-pov-ray__Mu92oJj", + "r1-t1800/crack-7z-hash__FVP3vRh", + "r1-t1800/dna-assembly__X4yEChU", + "r1-t1800/feal-differential-cryptanalysis__Hr4kPkt", + "r1-t1800/feal-linear-cryptanalysis__LLRkaNf", + "r1-t1800/filter-js-from-html__Q9y3YR3", + "r1-t1800/mailman__ZyAANtB", + "r1-t1800/make-mips-interpreter__W2F6ySm", + "r1-t1800/mcmc-sampling-stan__YoaMuqy", + "r1-t1800/mteb-retrieve__3ptj2AR", + "r1-t1800/path-tracing__HJxWpkM", + "r1-t1800/protein-assembly__bVCDQPN", + "r1-t2400/compile-compcert__z8xLH5e", + "r1-t3600/circuit-fibsqrt__66WdFy2", + "r1-t3600/distribution-search__oum44Cc", + "r1-t3600/fix-ocaml-gc__uj5TJYh", + "r1-t3600/install-windows-3.11__osH4pRk", + "r1-t3600/mteb-leaderboard__inv3Epe", + "r1-t750/overfull-hbox__ee9c4jM", + "r1-t900/adaptive-rejection-sampler__PArvFEg", + "r1-t900/build-cython-ext__eVWTonP", + "r1-t900/db-wal-recovery__qnwnRjY", + "r1-t900/fix-git__tPmQCGY", + "r1-t900/gcode-to-text__tyJktT9", + "r1-t900/gpt2-codegolf__7kdzjfU", + "r1-t900/headless-terminal__rA5DdVx", + "r1-t900/kv-store-grpc__vuLhgEt", + "r1-t900/make-doom-for-mips__BCT85A2", + "r1-t900/merge-diff-arc-agi-task__2rxMpAh", + "r1-t900/multi-source-data-merger__Y6TSdLB", + "r1-t900/openssl-selfsigned-cert__taa9Bhx", + "r1-t900/password-recovery__4Y3Xr67", + "r1-t900/polyglot-c-py__tHhKQ5i", + "r1-t900/polyglot-rust-c__4jKjy4e", + "r1-t900/prove-plus-comm__W6wjt7b", + "r1-t900/pytorch-model-cli__SnGhQgT", + "r1-t900/pytorch-model-recovery__fUqzL2D", + "r1-t900/query-optimize__bmmaRnM", + "r1-t900/raman-fitting__oN6bZVM", + "r1-t900/regex-log__gT67tpE", + "r1-t900/sparql-university__eeAPKyT", + "r1-t900/sqlite-with-gcov__vPgwNLw", + "r1-t900/tune-mjcf__mocj949", + "r3-t1200/code-from-image__DAaYNdw", + "r3-t1200/code-from-image__Te2W2Q9", + "r3-t1200/code-from-image__gft5KDm", + "r3-t12000/build-pov-ray__Qo6eLfg", + "r3-t12000/build-pov-ray__WNR9rdw", + "r3-t12000/build-pov-ray__X6Sq7bk", + "r3-t1800/crack-7z-hash__EnisgGH", + "r3-t1800/crack-7z-hash__GBpZR25", + "r3-t1800/crack-7z-hash__MbUQjgc", + "r3-t1800/dna-assembly__snHPgL4", + "r3-t1800/dna-assembly__uZczH9n", + "r3-t1800/feal-differential-cryptanalysis__FucyjUu", + "r3-t1800/feal-differential-cryptanalysis__RcFHbqX", + "r3-t1800/feal-differential-cryptanalysis__noEe3qY", + "r3-t1800/feal-linear-cryptanalysis__LvKpnQG", + "r3-t1800/feal-linear-cryptanalysis__YJm4wj9", + "r3-t1800/feal-linear-cryptanalysis__ua3sLc7", + "r3-t1800/filter-js-from-html__SqHW8Jw", + "r3-t1800/filter-js-from-html__WFc7rXA", + "r3-t1800/filter-js-from-html__pcqMJAv", + "r3-t1800/mailman__5Txbugx", + "r3-t1800/mailman__MNiFx9c", + "r3-t1800/mailman__akb93wt", + "r3-t1800/make-mips-interpreter__RpYQ2XG", + "r3-t1800/make-mips-interpreter__gHkvYoE", + "r3-t1800/make-mips-interpreter__nMGEqA8", + "r3-t1800/mcmc-sampling-stan__2grZFD7", + "r3-t1800/mcmc-sampling-stan__LE2RGm3", + "r3-t1800/mcmc-sampling-stan__bkVPVFd", + "r3-t1800/mteb-retrieve__MtszDxs", + "r3-t1800/mteb-retrieve__cVgJJGT", + "r3-t1800/mteb-retrieve__yPibexH", + "r3-t1800/path-tracing__3P8c5eJ", + "r3-t1800/path-tracing__AZ2NaW7", + "r3-t1800/path-tracing__g32vhrn", + "r3-t1800/protein-assembly__9fXvMs8", + "r3-t1800/protein-assembly__9rWMuWL", + "r3-t2400/compile-compcert__3FUpnGN", + "r3-t2400/compile-compcert__8y9sHD7", + "r3-t2400/compile-compcert__ho6QX8C", + "r3-t3600/circuit-fibsqrt__HiEsydF", + "r3-t3600/circuit-fibsqrt__eP6oWWJ", + "r3-t3600/circuit-fibsqrt__tLnVfqd", + "r3-t3600/distribution-search__NkQ5dpe", + "r3-t3600/distribution-search__SpheBQt", + "r3-t3600/distribution-search__gg8S57L", + "r3-t3600/fix-ocaml-gc__DixbcMC", + "r3-t3600/fix-ocaml-gc__LAzLaAE", + "r3-t3600/fix-ocaml-gc__NLxdQwD", + "r3-t3600/install-windows-3.11__4recKxm", + "r3-t3600/install-windows-3.11__FWc6e4X", + "r3-t3600/install-windows-3.11__Tr7Ftdg", + "r3-t3600/mteb-leaderboard__4H2CZvZ", + "r3-t3600/mteb-leaderboard__N7R2fzz", + "r3-t3600/mteb-leaderboard__rXGENDs", + "r3-t750/overfull-hbox__4nUm7SC", + "r3-t750/overfull-hbox__YqMaoHV", + "r3-t750/overfull-hbox__nwPUQUF", + "r3-t900/adaptive-rejection-sampler__LrDGcCP", + "r3-t900/adaptive-rejection-sampler__j3gYRYy", + "r3-t900/adaptive-rejection-sampler__xezMuBL", + "r3-t900/build-cython-ext__6CmysLh", + "r3-t900/build-cython-ext__6K4WgKE", + "r3-t900/build-cython-ext__9FQm4A2", + "r3-t900/db-wal-recovery__GwatioF", + "r3-t900/db-wal-recovery__vPbhdHk", + "r3-t900/db-wal-recovery__xtS6cXe", + "r3-t900/fix-git__4kuz8Bf", + "r3-t900/fix-git__97bA6rv", + "r3-t900/fix-git__EVT5tcy", + "r3-t900/gcode-to-text__C9BEpmi", + "r3-t900/gcode-to-text__maZQUAo", + "r3-t900/gcode-to-text__vESW7WK", + "r3-t900/gpt2-codegolf__MwZ5vUx", + "r3-t900/gpt2-codegolf__uk4VPFE", + "r3-t900/gpt2-codegolf__ymxiZBG", + "r3-t900/headless-terminal__PSKUgve", + "r3-t900/headless-terminal__nGGGKt8", + "r3-t900/headless-terminal__vruErZs", + "r3-t900/kv-store-grpc__iLR9Ty8", + "r3-t900/kv-store-grpc__jnvKq3w", + "r3-t900/kv-store-grpc__oMLDERL", + "r3-t900/make-doom-for-mips__6QzjHSh", + "r3-t900/make-doom-for-mips__KFVcsPX", + "r3-t900/make-doom-for-mips__YxYGpeF", + "r3-t900/merge-diff-arc-agi-task__Eg7roT5", + "r3-t900/merge-diff-arc-agi-task__LaYRW7y", + "r3-t900/merge-diff-arc-agi-task__jN5ZQCk", + "r3-t900/multi-source-data-merger__M9NwcKU", + "r3-t900/multi-source-data-merger__zWo3vRU", + "r3-t900/multi-source-data-merger__zq9qagK", + "r3-t900/openssl-selfsigned-cert__Jvn2v9p", + "r3-t900/openssl-selfsigned-cert__ixwYdd3", + "r3-t900/openssl-selfsigned-cert__qHzdJvH", + "r3-t900/password-recovery__RxgWyqW", + "r3-t900/password-recovery__Tz22qSr", + "r3-t900/password-recovery__k354MXi", + "r3-t900/polyglot-c-py__PRGhezc", + "r3-t900/polyglot-c-py__nMBTWWS", + "r3-t900/polyglot-c-py__p8j9Tm8", + "r3-t900/polyglot-rust-c__7ZKNx9M", + "r3-t900/polyglot-rust-c__Nr2bBxF", + "r3-t900/polyglot-rust-c__bPy8wEP", + "r3-t900/prove-plus-comm__3V2ASFe", + "r3-t900/prove-plus-comm__TBJs3k3", + "r3-t900/prove-plus-comm__w8dYms3", + "r3-t900/pytorch-model-cli__iyVSYTp", + "r3-t900/pytorch-model-cli__mB9QiSE", + "r3-t900/pytorch-model-cli__tiomLMm", + "r3-t900/pytorch-model-recovery__59kU9vX", + "r3-t900/pytorch-model-recovery__QPGAsmr", + "r3-t900/pytorch-model-recovery__iaPeQHQ", + "r3-t900/qemu-alpine-ssh__6o7vQiZ", + "r3-t900/qemu-alpine-ssh__pVxoQeN", + "r3-t900/qemu-alpine-ssh__vnBLusx", + "r3-t900/query-optimize__J7Lnk9f", + "r3-t900/query-optimize__TiunRY9", + "r3-t900/query-optimize__faCipRr", + "r3-t900/raman-fitting__gYiYG3u", + "r3-t900/raman-fitting__gbKShkf", + "r3-t900/raman-fitting__vPQbMrz", + "r3-t900/regex-log__TAnazuQ", + "r3-t900/regex-log__nzJvFZd", + "r3-t900/regex-log__xYSEQSM", + "r3-t900/sparql-university__QWQxg3J", + "r3-t900/sparql-university__ZfKbYmZ", + "r3-t900/sparql-university__iUADnDm", + "r3-t900/sqlite-with-gcov__JKx9qun", + "r3-t900/sqlite-with-gcov__QTZT6y5", + "r3-t900/sqlite-with-gcov__aTQSY3F", + "r3-t900/tune-mjcf__2i7JA4A", + "r3-t900/tune-mjcf__DbVdudc", + "r3-t900/tune-mjcf__ZbSzHAx", + "task1-kv-store-grpc/kv-store-grpc__Sj4AjYz" + ] +} diff --git a/bench/pricing.json b/bench/pricing.json new file mode 100644 index 00000000..c0941080 --- /dev/null +++ b/bench/pricing.json @@ -0,0 +1,23 @@ +{ + "_note": "USD per million tokens, used by scripts/dashboard.py to estimate costs. Edit when prices change; the dashboard labels every cost as an estimate from this file.", + "_source": "Anthropic list prices as reported on 2026-10-15 (verify at platform.claude.com); OpenAI GPT-6 Astra figures from price aggregators on 2026-10-17, unverified against developers.openai.com/api/docs/pricing", + "claude-opus-5-5": { + "input": 4.0, + "output": 20.0, + "cache_read": 0.2, + "cache_write": 5.0 + }, + "claude-opus-5": { + "input": 5.0, + "output": 25.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "gpt-6-astra": { + "input": 10.0, + "output": 50.0, + "cache_read": 1.0, + "cache_write": 10.0 + }, + "_note_openai": "OpenAI has no separate cache-write charge: the uncached part of a prompt (reported by the adapter as cache_write_tokens) is billed at the input price." +} diff --git a/bench/profiles/anthropic-m3.yaml b/bench/profiles/anthropic-m3.yaml new file mode 100644 index 00000000..532428c6 --- /dev/null +++ b/bench/profiles/anthropic-m3.yaml @@ -0,0 +1,64 @@ +# Measurement profile: built-in anthropic.yaml with the M3 features switched on. +# Compare against profiles/anthropic.yaml on the dev split. +family: anthropic +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: str_replace +effort: high +# No temperature: current Claude models reject sampling parameters. +max_output_tokens: 64000 # thinking counts toward this limit +context_window: 1000000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: false # minutes-left footer, nudge, final call (M4; off until measured) + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: false # warn after loop_repeats identical calls and results (M4) + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: none # none | client | server (server: Anthropic clears old results) + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: true +stop: # stop check (M3; off until measured) + enabled: true + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [system, last_two_messages], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } +anthropic: + thinking_display: summarized + thinking_binding: error + refusal_fallback: true diff --git a/bench/profiles/anthropic-m4-nolessons.yaml b/bench/profiles/anthropic-m4-nolessons.yaml new file mode 100644 index 00000000..2f592908 --- /dev/null +++ b/bench/profiles/anthropic-m4-nolessons.yaml @@ -0,0 +1,65 @@ +# The M4 benchmark profile with the learned lessons switched off (only change: lessons: false). +# Measurement profile: built-in anthropic.yaml with the M3 and M4 features switched on. +# Compare against bench/profiles/anthropic-m3.yaml on the dev split. +family: anthropic +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: str_replace +effort: high +# No temperature: current Claude models reject sampling parameters. +max_output_tokens: 64000 # thinking counts toward this limit +context_window: 1000000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: true + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: true + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: server + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: false # append prompts/lessons.md (learned from dev runs) +snapshot: true +stop: # stop check (M3; off until measured) + enabled: true + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [system, last_two_messages], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } +anthropic: + thinking_display: summarized + thinking_binding: error + refusal_fallback: true diff --git a/bench/profiles/anthropic-m4.yaml b/bench/profiles/anthropic-m4.yaml new file mode 100644 index 00000000..1f093ca9 --- /dev/null +++ b/bench/profiles/anthropic-m4.yaml @@ -0,0 +1,64 @@ +# Measurement profile: built-in anthropic.yaml with the M3 and M4 features switched on. +# Compare against bench/profiles/anthropic-m3.yaml on the dev split. +family: anthropic +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: str_replace +effort: high +# No temperature: current Claude models reject sampling parameters. +max_output_tokens: 64000 # thinking counts toward this limit +context_window: 1000000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: true + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: true + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: server + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: true +stop: # stop check (M3; off until measured) + enabled: true + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [system, last_two_messages], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } +anthropic: + thinking_display: summarized + thinking_binding: error + refusal_fallback: true diff --git a/bench/profiles/openai-m4.yaml b/bench/profiles/openai-m4.yaml new file mode 100644 index 00000000..1a018c21 --- /dev/null +++ b/bench/profiles/openai-m4.yaml @@ -0,0 +1,63 @@ +# Benchmark profile for OpenAI models: profiles/openai.yaml with the M3/M4 features on +# (snapshot, stop check, time budget, loop detector, client-side masking). +family: openai +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: apply_patch +effort: high +# No temperature: reasoning models do not take sampling parameters. +max_output_tokens: 64000 # reasoning counts toward this limit +context_window: 400000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: true # minutes-left footer, nudge, final call (M4; off until measured) + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: true # warn after loop_repeats identical calls and results (M4) + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: client # none | client | server (server: Anthropic clears old results) + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: true # environment snapshot in the first message (M3; off until measured) +stop: # stop check (M3; off until measured) + enabled: true + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } +openai: + reasoning_summary: auto + prompt_cache_key: gitagent-v2 diff --git a/bench/rounds.json b/bench/rounds.json new file mode 100644 index 00000000..80cc9e83 --- /dev/null +++ b/bench/rounds.json @@ -0,0 +1,219 @@ +{ + "_note": "Groups jobs in bench/jobs into rounds so the dashboard can show the current round's numbers next to earlier ones. A job belongs to the first round whose glob matches its name. Rounds with \"counts\": false are plumbing checks and stay out of every accuracy figure. \"skip\" lists dev tasks the round left out on purpose; they are not reported as pending. gitagent's current round is the last listed round that has gitagent runs. \"changes\" and \"reasoning\" describe what a round did and why it scored as it did; \"lessons_commit\" is the commit whose prompts/lessons.md the round's binary carried (the dashboard reads it with git show). \"split\": \"heldout\" marks a round on the held-out tasks; it is scored on those tasks and never becomes gitagent's current dev round.", + "rounds": [ + { + "name": "checks", + "label": "Plumbing checks (scripted model, single-task smoke runs)", + "host": "macOS and optimus", + "jobs": [ + "dry-*", + "m?-fix-git", + "optimus-gitagent-smoke", + "task1-*", + "smoke3-*", + "debug-*", + "smoke-gpt6*", + "smoke-nolessons*", + "smoke4-*", + "tb4-*" + ], + "counts": false, + "model": null + }, + { + "name": "mac-oracle", + "label": "Oracle on the Mac (amd64 emulated under QEMU, then Rosetta)", + "host": "macOS + Colima", + "jobs": [ + "smoke-oracle*", + "oracle-*" + ], + "model": null + }, + { + "name": "r0", + "label": "Passes 1-4 on the Mac (amd64 emulated; prompt, shim and lessons changed between passes)", + "host": "macOS + Colima", + "commit": "a91af9d..f4a0a30", + "jobs": [ + "pass*" + ], + "lessons_commit": null, + "changes": [ + "First attempts at all 45 dev tasks, in four passes of 10-14 tasks", + "Ran on the owner's Mac: amd64 emulated under QEMU, then under Rosetta from pass 3", + "Fixed between passes: the shim's relaunch when gitagent dies early, and a check that refuses a stale binary", + "Lessons learned after each pass: 0 in the prompt at the start, 9 by the end" + ], + "reasoning": "76.7% mean per task, but not comparable with later rounds. Part of the failures were the machine's: under emulation the reference solutions themselves failed on several tasks (no AVX, compiler crashes, no nested VM). Two more were harness bugs fixed during the round (a relaunch bug and a stale binary). The code and lessons changed between passes, so the round mixes several versions of gitagent.", + "model": "claude-opus-5-5" + }, + { + "name": "optimus-oracle", + "label": "Oracle on optimus, all 89 tasks, plus reruns of its failures", + "host": "optimus", + "jobs": [ + "optimus-oracle-*" + ], + "model": null + }, + { + "name": "terminus-baseline", + "label": "Terminus 2 baseline, 1 run per task (stopped by the owner with 41 of 44 scored)", + "host": "optimus", + "jobs": [ + "baseline-terminus2-*" + ], + "skip": [ + "qemu-alpine-ssh" + ], + "changes": [ + "Harbor's reference agent Terminus 2, same model (Opus 5.5) and machine, 1 run per task", + "Stopped by the owner after 41 of 44 tasks were scored" + ], + "reasoning": "87.8% on the 41 tasks it scored, which matches the public Vals.ai figure for Opus 5.5 with Terminus 2 (87.64% on all 89 tasks), so this machine reproduces leaderboard conditions. Its failures included two runaway loops: on password-recovery and crack-7z-hash the model returned empty replies (most likely refusals) and Terminus resent the whole conversation hundreds of times, about $350 between them.", + "model": "claude-opus-5-5" + }, + { + "name": "r1", + "label": "Round 1 on optimus: gitagent at 480265d, M4 profile, 9 lessons, no changes (45 dev tasks x 3 runs)", + "host": "optimus", + "commit": "480265d", + "jobs": [ + "r1-*" + ], + "lessons_commit": "480265d", + "changes": [ + "Moved to optimus: native x86_64, 32 cores, Docker", + "Same code as the end of r0 (480265d), M4 profile, 9 lessons in the prompt", + "All 45 dev tasks, 3 runs each (135 trials), so single lucky or unlucky runs average out" + ], + "reasoning": "94.1% mean per task (96.2% without qemu-alpine-ssh), +8.9 points over Terminus 2 on the tasks both ran. Every task r0 failed for machine or early-bug reasons now passes, so most of the jump from r0 is the real hardware, not new skill. Remaining failures: qemu-alpine-ssh 0/3 (this machine cannot run it); filter-js-from-html 0/3 (every run kept clean files byte-identical, while the instruction allows the parser's normalised output, which the r0 pass produced); protein-assembly and make-doom-for-mips 2/3 (run-to-run variation).", + "model": "claude-opus-5-5" + }, + { + "name": "r2", + "label": "Round 2 on optimus: r1 plus the /tests guard fix and lessons relearned from r1's 135 trials (45 dev tasks x 1 run)", + "host": "optimus", + "commit": "97fc537", + "jobs": [ + "r2-*" + ], + "lessons_commit": "98b89c4", + "changes": [ + "/tests guard fix: a redirect such as 2>/dev/null no longer counts as a write to /tests (13 false refusals over r0 and r1, each costing a turn)", + "Lessons relearned from r1's 135 trials: 54 added, 255 upvoted, 26 edited; 107 stored, 10 in the prompt (7 new or reworded, 6 dropped or reworded)", + "1 run per task; compared task by task with r1" + ], + "reasoning": "93.3% (42/45), the same as r1's 94.1% within run-to-run noise: only two tasks differ, and both already varied in r1 (make-doom-for-mips 2/3 in r1, passed here; protein-assembly 2/3 in r1, failed here). The guard fix works: 0 false /tests refusals, against 20 in r1's 135 trials. The relearned lessons made runs slightly longer (14.8 turns against 13.0, $0.266 against $0.247 per trial) but faster in wall time (median 74 s against 82 s), with no measurable change in accuracy. filter-js-from-html still fails (0/4 on optimus): the new lessons did not change how it reads the formatting requirement. One run per task cannot separate a change of a point or two from noise.", + "model": "claude-opus-5-5" + }, + { + "name": "final-heldout", + "split": "heldout", + "label": "Held-out run: r2's code frozen (97fc537), Claude Opus 5.5, the 44 held-out tasks x 3 runs. First and only use of the held-out split for this code.", + "host": "optimus", + "commit": "97fc537", + "lessons_commit": "98b89c4", + "jobs": [ + "heldout-opus55-*" + ], + "changes": [ + "Code frozen at r2 (97fc537): M4 profile, effort high, the /tests guard fix, 10 lessons", + "The 44 held-out tasks, never run, read or learned from before", + "3 runs per task; reported as single-run and 3-run accuracy", + "Learning is off: the learner refuses held-out tasks and no lessons change" + ], + "reasoning": "95.5% mean per task over 3 runs (126/132 trials; 97.7% without qemu-startup, which this machine cannot run). Single runs: 95.5%, 93.2% and 97.7%. 40 of 44 tasks passed every run and 43 at least once; no timeouts. The held-out score is not lower than the dev rounds (r1 94.1%, r2 93.3%), so the lessons and tuning on dev did not overfit. Not always passed: qemu-startup 0/3 (machine: no nested VM), modernize-scientific-stack, video-processing and extract-moves-from-video 2/3 each. Above every published Terminal-Bench 2.1 result (top: 91.4%, and 87.6% for Opus 5.5 with Terminus 2), with the caveat that those cover all 89 tasks and this covers the 44 held-out ones. Run note: 15 trials first failed to start because Docker ran out of network address pools with 32 trials at once; Harbor's job resume re-ran them, the first time without the API key (resume does not reload --env-file) and then with it. Only the final, complete runs are counted.", + "model": "claude-opus-5-5", + "final": true + }, + { + "name": "r3", + "label": "Round 3 on optimus: r2's harness with the rewritten system prompt and apply_patch tool, same lessons (45 dev tasks x 3 runs)", + "host": "optimus", + "commit": "57f0dc2", + "jobs": [ + "r3-*" + ], + "lessons_commit": "98b89c4", + "changes": [ + "System prompt written anew: the same rules and finishing conditions as before in new wording, no header comment", + "apply_patch tool reimplemented; it is only offered by the OpenAI profile, so these runs do not exercise it", + "Lessons, profile (M4), model and effort unchanged from r2", + "3 runs per task; compared task by task with r1 (3 runs) and r2 (1 run)" + ], + "reasoning": "91.9% (124/135 trials; 93.9% without qemu-alpine-ssh), against r1's 94.1% over the same 135-trial design and r2's 93.3% over one run each. No task flipped: the two tasks that fail every run are unchanged (qemu-alpine-ssh is the machine, filter-js-from-html is 0/10 since r0), make-doom-for-mips went from 2/3 to 3/3, and the three trials that separate r3 from r1 are single lost runs on three tasks that were 3/3 before (dna-assembly, polyglot-c-py, raman-fitting), plus protein-assembly at 1/3 after 2/3 and 0/1. Those failed trials ended normally: 11 to 23 turns, no timeouts or loops, two of them after the stop check had rejected an early task_complete. That is the known pattern of ambiguous specs read differently from the checker, not a new failure mode. Cost and shape are the same as r1: $0.243 per trial against $0.247, 13.2 turns against 13.0, median 82 s in both, 92.9% of input tokens from the cache against 91.8%; stop rejections 75 against 59; 0 false /tests refusals as in r2. Reading: the rewritten prompt performs the same as the old one within run-to-run noise; a difference of a point or two either way cannot be excluded with 135 trials.", + "model": "claude-opus-5-5" + }, + { + "name": "heldout-r3", + "split": "heldout", + "label": "Held-out check of r3's code (57f0dc2): the rewritten prompt, Claude Opus 5.5, the 44 held-out tasks x 1 run", + "host": "optimus", + "commit": "57f0dc2", + "lessons_commit": "98b89c4", + "jobs": [ + "heldout2-opus55-*" + ], + "changes": [ + "Same model, profile, effort and lessons as the first held-out run; only the system prompt and apply_patch differ", + "One run per task: a sanity check against the first held-out run's single runs (95.5%, 93.2%, 97.7%), not a new final score", + "Learning is off: the learner refuses held-out tasks; the result is not used to change code" + ], + "reasoning": "40/44 (90.9%; 93.0% without qemu-startup, which this machine cannot run; est. cost $14.08). The first held-out run on the old prompt gave 93.2%, 95.5% and 97.7% in its three single runs, so this run sits one trial below that band: qemu-startup 0/1 (0/3 before), extract-moves-from-video and video-processing 0/1 (both 2/3 before), and dna-insert 0/1 (3/3 before), the only task that was solid before and failed here. On its own this is noise (Fisher p = 0.22 against the 126/132 record). Taken together with dev round r3 (124/135 against r1's 127/135), both measurements of the rewritten prompt point the same way: pooled 164/179 (91.6%) against 253/267 (94.8%) for the old prompt, p = 0.13. Not proof of a regression, but enough to measure further before quoting a held-out number for this prompt. Nothing from this run is used to change the prompt; any prompt work happens on the dev split.", + "model": "claude-opus-5-5" + }, + { + "name": "g1", + "model": "gpt-6-astra", + "label": "Round g1 on optimus: GPT-6 Astra, OpenAI M4 profile (apply_patch), same lessons as r3 (45 dev tasks x 1 run)", + "host": "optimus", + "commit": "57f0dc2", + "jobs": [ + "g1-*" + ], + "lessons_commit": "98b89c4", + "changes": [ + "First round on a second model family: openai/gpt-6-astra through the Responses API adapter", + "Profile bench/profiles/openai-m4.yaml: the OpenAI profile with snapshot, stop check, time budget, loop detector and client-side masking on, apply_patch as the edit tool, effort high", + "Same harness build (57f0dc2) and lessons as r3; 1 run per task, the same design as Claude's r2" + ], + "reasoning": "88.9% (40/45; 90.9% without qemu-alpine-ssh, which this machine cannot run), against Claude Opus 5.5 at 93.3% (r2, the same 1-run design) and 91.9% (r3, 3 runs) on the same tasks, harness build and lessons. GPT-6 Astra failed the two tasks every run fails (qemu-alpine-ssh, filter-js-from-html) and three that Opus passes every time (mailman, make-doom-for-mips, mteb-retrieve); it passed protein-assembly, which Opus is flaky on. All 45 trials ended normally: no timeouts, no loops, 33 stop-check rejections (it tried to finish early in about a third of the tasks). It works differently: 22.6 turns per trial against Opus's 13-15, median 141 s against 74-82 s, and $1.30 per trial (median $0.66; a few long tasks cost $3-7 each) against $0.24-0.27 (est. $58.60 for the round, including the re-run of pytorch-model-cli, whose checker timed out the first time and which then passed). apply_patch, used for the first time by a real model: 34 calls, 0 failures. This is a first round with a profile carried over from the Claude one and never tuned for GPT-6, and one run per task cannot separate a point or two from noise." + }, + { + "name": "g2", + "model": "gpt-6-astra", + "label": "Round g2 on optimus: g1 plus lessons relearned from r3 and g1 (45 dev tasks x 1 run)", + "host": "optimus", + "commit": "9775813", + "jobs": [ + "g2-*" + ], + "lessons_commit": "9775813", + "changes": [ + "Lessons relearned from r3's 135 and g1's 45 trials: 44 added, 24 edited, 396 upvoted; 151 stored, 10 in the prompt (4 new or reworded, 2 dropped from the top 10)", + "Same profile (bench/profiles/openai-m4.yaml), model and harness code as g1; 1 run per task, compared task by task with g1" + ], + "reasoning": "88.9% (40/45; 90.9% without qemu-alpine-ssh), the same as g1. Only two tasks changed, in opposite directions: make-doom-for-mips passed (failed in g1), dna-assembly failed (passed in g1, flaky for Opus too). The relearned lessons did not change accuracy but made GPT-6 finish more cleanly: stop-check rejections fell from 31 to 18 (-42%), turns per trial from 22.5 to 21.4, median time from 141 s to 126 s, and cost from $58.60 to $54.12 ($1.20 per trial). apply_patch: 30 calls, 1 failure. Still failing: qemu-alpine-ssh (machine), filter-js-from-html (every model), mailman and mteb-retrieve (pass for Opus every time)." + }, + { + "name": "heldout-g2", + "split": "heldout", + "model": "gpt-6-astra", + "label": "Held-out run of g2's code (9775813): GPT-6 Astra, the 44 held-out tasks x 1 run", + "host": "optimus", + "commit": "9775813", + "lessons_commit": "9775813", + "jobs": [ + "heldout-gpt6-*" + ], + "changes": [ + "Code, lessons and profile (bench/profiles/openai-m4.yaml) frozen at g2", + "One run per task; more runs can be added later as further jobs", + "Learning is off: the learner refuses held-out tasks; the result is not used to change code" + ], + "reasoning": "37/44 (84.1%; 86.0% without qemu-startup, which this machine cannot run; est. cost $51.72). Two of the seven failures were not attempts: on break-filter-js-from-html and vulnerable-secret, OpenAI's API stopped the run with a cyber_policy refusal (content flagged for possible cybersecurity risk; OpenAI offers a separate access programme for authorised security work). Without those two the model solved 37 of 42 (88.1%); without them and qemu-startup, 37 of 41 (90.2%). The other failures: qemu-startup (machine; 0/3 for Opus too), video-processing (2/3 for Opus), and three that Opus solved in all three of its held-out runs: extract-elf, sanitize-git-repo, torch-pipeline-parallelism. One run per task, so about +/-3 points of noise; lower than GPT-6's dev rounds (88.9%), with the refusals accounting for most of the gap. Not comparable to the official leaderboard (89 tasks x 5 runs), and not used to change code." + } + ] +} diff --git a/bench/split.json b/bench/split.json new file mode 100644 index 00000000..c02bd1c7 --- /dev/null +++ b/bench/split.json @@ -0,0 +1,97 @@ +{ + "dataset": "terminal-bench/terminal-bench-2-1", + "method": "stratified by task.toml category and difficulty; seed 20261015; forced dev: fix-git, kv-store-grpc", + "dev": [ + "adaptive-rejection-sampler", + "build-cython-ext", + "build-pov-ray", + "circuit-fibsqrt", + "code-from-image", + "compile-compcert", + "crack-7z-hash", + "db-wal-recovery", + "distribution-search", + "dna-assembly", + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis", + "filter-js-from-html", + "fix-git", + "fix-ocaml-gc", + "gcode-to-text", + "gpt2-codegolf", + "headless-terminal", + "install-windows-3.11", + "kv-store-grpc", + "mailman", + "make-doom-for-mips", + "make-mips-interpreter", + "mcmc-sampling-stan", + "merge-diff-arc-agi-task", + "mteb-leaderboard", + "mteb-retrieve", + "multi-source-data-merger", + "openssl-selfsigned-cert", + "overfull-hbox", + "password-recovery", + "path-tracing", + "polyglot-c-py", + "polyglot-rust-c", + "protein-assembly", + "prove-plus-comm", + "pytorch-model-cli", + "pytorch-model-recovery", + "qemu-alpine-ssh", + "query-optimize", + "raman-fitting", + "regex-log", + "sparql-university", + "sqlite-with-gcov", + "tune-mjcf" + ], + "heldout": [ + "bn-fit-modify", + "break-filter-js-from-html", + "build-pmars", + "caffe-cifar-10", + "cancel-async-tasks", + "chess-best-move", + "cobol-modernization", + "configure-git-webserver", + "constraints-scheduling", + "count-dataset-tokens", + "custom-memory-heap-crash", + "dna-insert", + "extract-elf", + "extract-moves-from-video", + "financial-document-processor", + "fix-code-vulnerability", + "git-leak-recovery", + "git-multibranch", + "hf-model-inference", + "large-scale-text-editing", + "largest-eigenval", + "llm-inference-batching-scheduler", + "log-summary-date-ranges", + "model-extraction-relu-logits", + "modernize-scientific-stack", + "nginx-request-logging", + "path-tracing-reverse", + "portfolio-optimization", + "pypi-server", + "qemu-startup", + "regex-chess", + "reshard-c4-data", + "rstan-to-pystan", + "sam-cell-seg", + "sanitize-git-repo", + "schemelike-metacircular-eval", + "sqlite-db-truncate", + "torch-pipeline-parallelism", + "torch-tensor-parallelism", + "train-fasttext", + "video-processing", + "vulnerable-secret", + "winning-avg-corewars", + "write-compressor" + ] +} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml new file mode 100644 index 00000000..71367b84 --- /dev/null +++ b/crates/cli/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "gitagent-cli" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[[bin]] +name = "gitagent" +path = "src/main.rs" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +gitagent-core.workspace = true +gitagent-learn.workspace = true +gitagent-loader.workspace = true +gitagent-provider.workspace = true +gitagent-record.workspace = true +gitagent-repo.workspace = true +gitagent-sandbox.workspace = true +gitagent-sdk.workspace = true +gitagent-tools.workspace = true +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls", "http2"] } +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +serde_json.workspace = true +tempfile = "3.27.0" diff --git a/crates/cli/src/learn_cmd.rs b/crates/cli/src/learn_cmd.rs new file mode 100644 index 00000000..1a080d75 --- /dev/null +++ b/crates/cli/src/learn_cmd.rs @@ -0,0 +1,155 @@ +//! `gitagent learn`: distils lessons from finished benchmark trials. + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use anyhow::{bail, Result}; +use clap::Parser; +use gitagent_core::{Outcome, RunReport}; +use gitagent_learn::LearnOptions; +use gitagent_loader::LoadOptions; + +use crate::Setup; + +#[derive(Debug, Parser)] +#[command( + name = "gitagent learn", + about = "Distil general lessons from gitagent's own finished trials into the lessons file.", + long_about = "Reads each trial's agent/trajectory.json and the reward in its result.json \ + (it never reads tests or verifier output), asks the model for lesson \ + changes, rejects any that carry task details, and rewrites the lessons \ + file. Review the diff before committing it. Learn from the dev split only." +)] +struct LearnCli { + /// A Harbor job directory to learn from (repeatable). + #[arg(long = "from", required = true)] + jobs: Vec, + + /// Split file whose `dev` list names the tasks that may be learned from. + #[arg(long, default_value = "bench/split.json")] + split: PathBuf, + + /// Learn from every trial because no split exists yet (only for tasks + /// that will go in the dev split). + #[arg(long)] + no_split: bool, + + /// The lesson store with importance counts. + #[arg(long, default_value = "bench/lessons/lessons.json")] + store: PathBuf, + + /// The lessons file to rewrite. + #[arg(long, default_value = "prompts/lessons.md")] + lessons: PathBuf, + + /// Size limit of the lessons file in characters. + #[arg(long, default_value_t = 3000)] + max_chars: usize, + + /// Show a lesson to the agent only once attempts at this many different + /// tasks support it. + #[arg(long, default_value_t = 2)] + min_tasks: usize, + + /// Model as provider:model. + #[arg(long)] + model: Option, + + /// Profile: auto, a built-in name, or a path to a profile YAML file. + #[arg(long)] + profile: Option, + + /// Show what would change without writing anything. + #[arg(long)] + dry_run: bool, +} + +pub(crate) async fn run(args: &[std::ffi::OsString]) -> ExitCode { + let cli = LearnCli::parse_from( + std::iter::once(std::ffi::OsString::from("gitagent learn")).chain(args.iter().cloned()), + ); + match learn(cli).await { + Ok(()) => ExitCode::SUCCESS, + Err(e) => crate::usage_error(e), + } +} + +async fn learn(cli: LearnCli) -> Result<()> { + let dev_tasks = match (cli.split.is_file(), cli.no_split) { + (true, _) => Some(gitagent_learn::read_split(&cli.split)?), + (false, true) => None, + (false, false) => bail!( + "{} not found: learning needs the dev split; pass --no-split only for tasks that will be dev", + cli.split.display() + ), + }; + // Built-in configuration only, as in --bench. + let spec = gitagent_loader::load(&LoadOptions { + model: cli.model.clone(), + profile: cli.profile.clone(), + ..Default::default() + })?; + let provider = crate::make_provider(&spec)?; + eprintln!( + "gitagent learn: model {} · {} job(s)", + spec.model, + cli.jobs.len() + ); + let report = gitagent_learn::learn( + &spec, + provider.as_ref(), + &LearnOptions { + jobs: cli.jobs, + store_path: cli.store, + lessons_path: cli.lessons.clone(), + dev_tasks, + max_chars: cli.max_chars, + min_tasks: cli.min_tasks, + dry_run: cli.dry_run, + }, + ) + .await?; + println!("{}", serde_json::to_string_pretty(&report)?); + if cli.dry_run { + println!("\nLessons after this pass:\n{}", report.rendered); + } else { + eprintln!( + "gitagent learn: {} lessons stored, {} shown; review the diff of {} before committing", + report.lessons_total, + report.lessons_shown, + cli.lessons.display() + ); + } + Ok(()) +} + +/// Online learning after a finished task: with `learning: online`, outside +/// `--bench`, and only when the agent completed the task. Returns whether +/// lessons were learned. A failure is reported but does not change the run. +pub(crate) async fn after_run(s: &Setup, run: &RunReport, project_dir: &Path, bench: bool) -> bool { + if bench || s.spec.config.learning != "online" || run.outcome != Outcome::Completed { + return false; + } + let result = gitagent_learn::learn_from_run( + &s.spec, + s.provider.as_ref(), + &run.trajectory_path, + project_dir, + ) + .await; + match result { + Ok(r) => { + s.events + .emit("learn", serde_json::to_value(&r).unwrap_or_default()); + eprintln!( + "gitagent: learned {} new, {} upvoted, {} rejected · {} lessons stored, {} in .gitagent/prompts/lessons.md", + r.lessons_added, r.lessons_upvoted, r.ops_rejected, r.lessons_total, r.lessons_shown + ); + r.trials_learned > 0 + } + Err(e) => { + eprintln!("gitagent: learning from this run failed: {e:#}"); + false + } + } +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs new file mode 100644 index 00000000..97534cc2 --- /dev/null +++ b/crates/cli/src/main.rs @@ -0,0 +1,467 @@ +mod learn_cmd; +mod otel; +mod repl; +mod repo_mode; +mod sandbox_exec; + +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use clap::{Parser, ValueEnum}; +use gitagent_core::{Finished, Outcome, RunOptions}; +use gitagent_loader::{AgentSpec, LoadOptions}; +use gitagent_provider::Provider; +use gitagent_record::Events; + +#[derive(Debug, Parser)] +#[command( + name = "gitagent", + version, + about = "A terminal agent whose behavior lives in version-controlled files.", + after_help = "Without -p or --instruction-file, gitagent starts an interactive session in --dir.\n\ + `gitagent learn --help` describes learning lessons from finished trials." +)] +pub(crate) struct Cli { + /// Benchmark mode: built-in config only, no git commits, no memory. + #[arg(long)] + bench: bool, + + /// Read the task from this file. + #[arg(long, conflicts_with = "prompt")] + instruction_file: Option, + + /// The task, given inline. + #[arg(short = 'p', long)] + prompt: Option, + + /// Model as provider:model (overrides gitagent.yaml). + #[arg(long)] + model: Option, + + /// Profile: auto, a built-in name, or a path to a profile YAML file. + #[arg(long)] + profile: Option, + + /// Path to gitagent.yaml; its prompts/ and profiles/ override the built-ins. + #[arg(long)] + config: Option, + + /// Directory the agent works in (default: current directory). + #[arg(long)] + dir: Option, + + /// Where the trajectory, counters and logs are written. + #[arg(long, default_value = "/tmp/gitagent")] + out_dir: PathBuf, + + /// Wall-clock budget for the run in seconds (default: timeout_s from the config). + #[arg(long)] + timeout_s: Option, + + /// Session id recorded in the trajectory. + #[arg(long)] + session_id: Option, + + /// Write events to stdout, one JSON object per line. + #[arg(long, value_enum)] + events: Option, + + /// `text` prints the agent's summary; `stream-json` prints the SDK's + /// messages (system, assistant, user, result), one per line. + #[arg(long, value_enum, default_value = "text")] + output_format: OutputFormat, + + /// Stop after this many model calls. + #[arg(long)] + max_turns: Option, + + /// Offer only these tools, comma-separated (`.claude/` or gitagent names). + #[arg(long, value_delimiter = ',')] + allowed_tools: Vec, + + /// Never offer these tools, comma-separated (`.claude/` or gitagent names). + #[arg(long, value_delimiter = ',')] + disallowed_tools: Vec, + + /// Text added after the system prompt. + #[arg(long)] + append_system_prompt: Option, + + /// Clone this repository, work on a new branch and commit the result. + #[arg(long)] + repo: Option, + + /// Branch to start from in repo mode (default: the remote's default branch). + #[arg(long, requires = "repo")] + base: Option, + + /// Work on this branch in repo mode; an existing remote branch is resumed. + #[arg(long, requires = "repo")] + branch: Option, + + /// Push the working branch after committing (never forced). + #[arg(long, requires = "repo")] + push: bool, + + /// After the run, keep the shells open (and anything still running in + /// them) until they exit, instead of stopping them. For harnesses that + /// check the environment after the agent ends; they can wait for the + /// `done` file in the output directory. + #[arg(long)] + linger: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum EventFormat { + Jsonl, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum OutputFormat { + Text, + StreamJson, +} + +/// What every mode needs: the loaded configuration and a provider. +pub(crate) struct Setup { + pub spec: AgentSpec, + pub provider: Box, + pub work_dir: PathBuf, + pub timeout: Duration, + pub events: Events, + /// Directories the sandboxed shell may write to; `None` when the + /// sandbox is off. + pub sandbox: Option>, +} + +fn main() -> ExitCode { + // The sandbox launcher runs before anything else and replaces itself + // with the shell. + let args: Vec = std::env::args_os().collect(); + if args.get(1).and_then(|a| a.to_str()) == Some("__sandbox") { + return sandbox_exec::run(&args[2..]); + } + let learn = args.get(1).and_then(|a| a.to_str()) == Some("learn"); + match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(rt) if learn => rt.block_on(learn_cmd::run(&args[2..])), + Ok(rt) => rt.block_on(async_main()), + Err(e) => usage_error(e.into()), + } +} + +async fn async_main() -> ExitCode { + let cli = Cli::parse(); + let task = match task_from(&cli) { + Ok(t) => t, + Err(e) => return usage_error(e), + }; + if cli.repo.is_some() && cli.bench { + return usage_error(anyhow::anyhow!("--repo cannot be used with --bench")); + } + match task { + Some(instruction) if cli.repo.is_some() => { + let url = cli.repo.clone().unwrap_or_default(); + repo_mode::run(cli, url, instruction).await + } + None if cli.repo.is_some() => usage_error(anyhow::anyhow!( + "--repo needs a task: give -p/--prompt or --instruction-file" + )), + Some(instruction) => single(cli, instruction).await, + None if cli.bench => usage_error(anyhow::anyhow!( + "--bench needs a task: give --instruction-file or -p/--prompt" + )), + None => match setup(&cli) { + Ok(s) => match repl::run(s, &cli.out_dir).await { + Ok(()) => ExitCode::SUCCESS, + Err(e) => usage_error(e), + }, + Err(e) => usage_error(e), + }, + } +} + +pub(crate) fn usage_error(e: anyhow::Error) -> ExitCode { + eprintln!("gitagent: {e:#}"); + ExitCode::from(2) +} + +fn task_from(cli: &Cli) -> Result> { + let task = match (&cli.instruction_file, &cli.prompt) { + (Some(path), None) => Some( + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?, + ), + (None, Some(p)) => Some(p.clone()), + _ => None, + }; + if task.as_deref().is_some_and(|t| t.trim().is_empty()) { + bail!("the task is empty"); + } + Ok(task) +} + +pub(crate) fn setup(cli: &Cli) -> Result { + let work_dir = match &cli.dir { + Some(d) => d.clone(), + None => std::env::current_dir().context("reading the current directory")?, + }; + // Benchmark runs use only the built-in files and explicit flags, so the + // task's directory or the machine's home cannot change the agent. + let (project_dir, home_dir) = if cli.bench { + (None, None) + } else { + ( + Some(work_dir.clone()), + std::env::var_os("HOME").map(PathBuf::from), + ) + }; + let spec = gitagent_loader::load(&LoadOptions { + config_path: cli.config.clone(), + model: cli.model.clone(), + profile: cli.profile.clone(), + project_dir, + home_dir, + })?; + if !matches!(spec.config.learning.as_str(), "off" | "online") { + bail!( + "learning must be off or online, got {:?}", + spec.config.learning + ); + } + let mut spec = spec; + if let Some(n) = cli.max_turns { + spec.config.max_turns = n; + } + if let Some(extra) = &cli.append_system_prompt { + spec.system_prompt.push('\n'); + spec.system_prompt.push_str(extra.trim_end()); + spec.system_prompt.push('\n'); + } + let provider = make_provider(&spec)?; + let timeout = Duration::from_secs(cli.timeout_s.unwrap_or(spec.config.timeout_s)); + let events = match cli.events { + Some(EventFormat::Jsonl) => Events::stdout(), + None => Events::disabled(), + }; + let sandbox = sandbox_dirs(&spec.config.sandbox, cli.bench, &work_dir, &cli.out_dir)?; + eprintln!( + "gitagent {}: model {} · config {} · profile {} · {}{} · sandbox {}", + env!("CARGO_PKG_VERSION"), + spec.model, + spec.config_source, + spec.profile_source, + work_dir.display(), + if cli.bench { " · bench" } else { "" }, + if sandbox.is_some() { "on" } else { "off" } + ); + Ok(Setup { + spec, + provider, + work_dir, + timeout, + events, + sandbox, + }) +} + +/// One task (`-p`, `--instruction-file`, and `--bench`). +async fn single(cli: Cli, instruction: String) -> ExitCode { + let out_dir = cli.out_dir.clone(); + let result = async { + let s = setup(&cli)?; + let stream_json = cli.output_format == OutputFormat::StreamJson; + let session_id = cli + .session_id + .clone() + .unwrap_or_else(|| format!("gitagent-{}", std::process::id())); + let tool_allowlist = (!cli.allowed_tools.is_empty()) + .then(|| gitagent_loader::sdk::normalize_tool_list(&cli.allowed_tools)); + let tool_denylist = gitagent_loader::sdk::normalize_tool_list(&cli.disallowed_tools); + // Stream output: gitagent's events are converted as they happen. + let converter = stream_json.then(|| { + std::sync::Arc::new(std::sync::Mutex::new(gitagent_sdk::Converter::new( + &session_id, + &s.spec.model.to_string(), + u64::from(s.spec.profile.context_window), + None, + ))) + }); + let events = match &converter { + Some(c) => { + let init = c.lock().unwrap().init( + &s.work_dir, + &tool_names(&s.spec, tool_allowlist.as_deref(), &tool_denylist), + &s.spec + .agents + .iter() + .map(|a| a.name.clone()) + .collect::>(), + &s.spec + .skills + .iter() + .map(|k| k.name.clone()) + .collect::>(), + ); + println!("{}", serde_json::to_string(&init)?); + let c = c.clone(); + Events::to_callback(move |event| { + for m in c.lock().unwrap().feed(&event) { + if let Ok(line) = serde_json::to_string(&m) { + println!("{line}"); + } + } + }) + } + None => s.events.clone(), + }; + let finished = gitagent_core::run( + &s.spec, + s.provider.as_ref(), + RunOptions { + instruction, + work_dir: s.work_dir.clone(), + out_dir: out_dir.clone(), + timeout: Some(s.timeout), + session_id: Some(session_id), + events, + resume: None, + sandbox: s.sandbox.clone(), + tool_allowlist, + tool_denylist, + hook_callbacks: None, + }, + ) + .await?; + report(&finished, !s.events.enabled() && !stream_json); + if !cli.bench { + export_telemetry(&finished, &s.spec).await; + } + learn_cmd::after_run(&s, &finished.report, &s.work_dir, cli.bench).await; + anyhow::Ok(finished) + } + .await; + + let (code, finished) = match result { + // Running out of turns or time is a normal end of a run; only a + // provider failure gets its own exit code so harnesses can tell it apart. + Ok(f) => ( + if matches!(f.report.outcome, Outcome::ProviderFailed(_)) { + 3 + } else { + 0 + }, + Some(f), + ), + Err(e) => { + eprintln!("gitagent: {e:#}"); + (2, None) + } + }; + // Written last, after the trajectory and counters: a harness waiting for + // this file can read everything else. + if let Err(e) = std::fs::create_dir_all(&out_dir) + .and_then(|_| std::fs::write(out_dir.join("done"), format!("{code}\n"))) + { + eprintln!("gitagent: writing {}: {e}", out_dir.join("done").display()); + } + if let Some(mut f) = finished.filter(|_| cli.linger) { + eprintln!("gitagent: done; keeping the shells open until they exit"); + f.terminal.linger().await; + } + ExitCode::from(code) +} + +/// The tools a run offers after the allow and deny lists. +fn tool_names(spec: &AgentSpec, allow: Option<&[String]>, deny: &[String]) -> Vec { + let mut tools: Vec = gitagent_tools::specs(&spec.profile.tools, spec.profile.edit_tool) + .into_iter() + .map(|t| t.name) + .collect(); + if !spec.skills.is_empty() { + tools.push(gitagent_tools::SKILL.into()); + } + if !spec.agents.is_empty() { + tools.push(gitagent_tools::SUBAGENT.into()); + } + if let Some(allow) = allow { + tools.retain(|t| allow.contains(t) || t == gitagent_tools::TASK_COMPLETE); + } + tools.retain(|t| !deny.contains(t) || t == gitagent_tools::TASK_COMPLETE); + tools +} + +/// Sends traces and metrics when `OTEL_EXPORTER_OTLP_ENDPOINT` is set. A +/// failure is reported but does not change the run's result. +pub(crate) async fn export_telemetry(f: &Finished, spec: &AgentSpec) { + if let Some(exporter) = otel::Exporter::from_env() { + if let Err(e) = exporter.export(&f.report, &spec.model.to_string()).await { + eprintln!("gitagent: telemetry export failed: {e:#}"); + } + } +} + +/// Prints the outcome: a status line on stderr and, unless events go to +/// stdout, the agent's summary on stdout. +pub(crate) fn report(f: &Finished, summary_to_stdout: bool) { + let r = &f.report; + eprintln!( + "gitagent: {} after {} turns · tokens in {} (cached {}) out {} · {}", + r.outcome, + r.turns, + r.usage.input_tokens, + r.usage.cache_read_tokens, + r.usage.output_tokens, + r.trajectory_path.display() + ); + if summary_to_stdout { + match &r.summary { + Some(s) => println!("{s}"), + None => println!("(no summary: the run ended with {})", r.outcome), + } + } +} + +/// `sandbox: auto` means on outside `--bench` (the benchmark container is the +/// isolation there) wherever the sandbox is available; `on` and `off` force it. +fn sandbox_dirs( + setting: &str, + bench: bool, + work_dir: &std::path::Path, + out_dir: &std::path::Path, +) -> Result>> { + let on = match setting { + "auto" => !bench && gitagent_sandbox::available(), + "on" => { + if !gitagent_sandbox::available() { + bail!("sandbox: on needs Linux (Landlock and seccomp)"); + } + true + } + "off" => false, + other => bail!("sandbox must be auto, on or off, got {other:?}"), + }; + if !on { + return Ok(None); + } + let mut dirs = vec![ + work_dir.to_path_buf(), + out_dir.to_path_buf(), + std::env::temp_dir(), + PathBuf::from("/tmp"), + PathBuf::from("/var/tmp"), + PathBuf::from("/dev"), + ]; + if let Some(home) = std::env::var_os("HOME") { + dirs.push(PathBuf::from(home).join(".cache")); + } + std::fs::create_dir_all(out_dir)?; + Ok(Some(dirs)) +} + +pub(crate) fn make_provider(spec: &AgentSpec) -> Result> { + gitagent_provider::for_spec(spec) +} diff --git a/crates/cli/src/otel.rs b/crates/cli/src/otel.rs new file mode 100644 index 00000000..40a0e9fb --- /dev/null +++ b/crates/cli/src/otel.rs @@ -0,0 +1,331 @@ +//! OpenTelemetry export, off unless `OTEL_EXPORTER_OTLP_ENDPOINT` is set (and +//! never in `--bench`). After a run, the trajectory and counters become OTLP +//! traces (a run span, a span per turn and per tool call) and metrics (the +//! counters), posted as OTLP/HTTP JSON. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{bail, Context, Result}; +use gitagent_core::RunReport; +use serde_json::{json, Value}; + +pub(crate) struct Exporter { + endpoint: String, + headers: Vec<(String, String)>, + service: String, +} + +impl Exporter { + pub fn from_env() -> Option { + let endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT") + .ok() + .filter(|e| !e.trim().is_empty())?; + let headers = std::env::var("OTEL_EXPORTER_OTLP_HEADERS") + .unwrap_or_default() + .split(',') + .filter_map(|kv| kv.split_once('=')) + .map(|(k, v)| (k.trim().to_string(), v.trim().to_string())) + .collect(); + Some(Exporter { + endpoint: endpoint.trim_end_matches('/').to_string(), + headers, + service: std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "gitagent".into()), + }) + } + + pub async fn export(&self, report: &RunReport, model: &str) -> Result<()> { + let trajectory: Value = serde_json::from_slice( + &std::fs::read(&report.trajectory_path).context("reading the trajectory")?, + )?; + let traces = traces(&trajectory, report, model, &self.service, now_nanos()); + let metrics = metrics(report, &self.service, now_nanos()); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + for (path, body) in [("v1/traces", traces), ("v1/metrics", metrics)] { + let mut req = client + .post(format!("{}/{path}", self.endpoint)) + .header("content-type", "application/json") + .body(body.to_string()); + for (k, v) in &self.headers { + req = req.header(k, v); + } + let resp = req.send().await?; + if !resp.status().is_success() { + bail!("{path}: HTTP {}", resp.status()); + } + } + Ok(()) + } +} + +fn resource(service: &str) -> Value { + json!({"attributes": [ + attr("service.name", json!({"stringValue": service})), + attr("service.version", json!({"stringValue": env!("CARGO_PKG_VERSION")})), + ]}) +} + +fn attr(key: &str, value: Value) -> Value { + json!({"key": key, "value": value}) +} + +fn int(v: u64) -> Value { + json!({"intValue": v.to_string()}) +} + +fn text(v: &str) -> Value { + json!({"stringValue": v}) +} + +pub(crate) fn traces(t: &Value, report: &RunReport, model: &str, service: &str, now: u64) -> Value { + let trace_id = hex_id(16, now); + let root_id = hex_id(8, now ^ 1); + let steps = t["steps"].as_array().cloned().unwrap_or_default(); + let times: Vec = steps + .iter() + .map(|s| { + s["timestamp"] + .as_str() + .and_then(parse_rfc3339) + .unwrap_or(now) + }) + .collect(); + let start = times.first().copied().unwrap_or(now); + let mut spans = vec![json!({ + "traceId": trace_id, + "spanId": root_id, + "name": "gitagent.run", + "kind": 1, + "startTimeUnixNano": start.to_string(), + "endTimeUnixNano": now.to_string(), + "attributes": [ + attr("gen_ai.request.model", text(model)), + attr("gitagent.outcome", text(&report.outcome.to_string())), + attr("gitagent.turns", int(report.turns.into())), + attr("gen_ai.usage.input_tokens", int(report.usage.input_tokens)), + attr("gen_ai.usage.output_tokens", int(report.usage.output_tokens)), + attr("gitagent.usage.cache_read_tokens", int(report.usage.cache_read_tokens)), + ], + })]; + let mut turn = 0u64; + for (i, step) in steps.iter().enumerate() { + if step["source"] != "agent" { + continue; + } + turn += 1; + // A step runs from the previous step's timestamp to its own. + let begin = if i > 0 { times[i - 1] } else { start }; + let end = times[i].max(begin); + let turn_id = hex_id(8, now ^ (turn << 8)); + let m = &step["metrics"]; + spans.push(json!({ + "traceId": trace_id, + "spanId": turn_id, + "parentSpanId": root_id, + "name": "gitagent.turn", + "kind": 1, + "startTimeUnixNano": begin.to_string(), + "endTimeUnixNano": end.to_string(), + "attributes": [ + attr("gitagent.turn", int(turn)), + attr("gen_ai.usage.input_tokens", int(m["prompt_tokens"].as_u64().unwrap_or(0))), + attr("gen_ai.usage.output_tokens", int(m["completion_tokens"].as_u64().unwrap_or(0))), + attr("gitagent.usage.cache_read_tokens", int(m["cached_tokens"].as_u64().unwrap_or(0))), + ], + })); + for (k, call) in step["tool_calls"] + .as_array() + .into_iter() + .flatten() + .enumerate() + { + spans.push(json!({ + "traceId": trace_id, + "spanId": hex_id(8, now ^ (turn << 8) ^ (k as u64 + 1) << 32), + "parentSpanId": turn_id, + "name": "gitagent.tool", + "kind": 1, + "startTimeUnixNano": begin.to_string(), + "endTimeUnixNano": end.to_string(), + "attributes": [attr("gen_ai.tool.name", text(call["function_name"].as_str().unwrap_or("")))], + })); + } + } + json!({"resourceSpans": [{ + "resource": resource(service), + "scopeSpans": [{"scope": {"name": "gitagent"}, "spans": spans}], + }]}) +} + +pub(crate) fn metrics(report: &RunReport, service: &str, now: u64) -> Value { + let mut out = Vec::new(); + if let Value::Object(counters) = report.counters.to_json() { + for (name, value) in counters { + let points: Vec = match &value { + Value::Number(n) => vec![json!({ + "asInt": n.as_u64().unwrap_or(0).to_string(), + "timeUnixNano": now.to_string(), + })], + Value::Object(labels) => labels + .iter() + .map(|(label, v)| { + json!({ + "asInt": v.as_u64().unwrap_or(0).to_string(), + "timeUnixNano": now.to_string(), + "attributes": [attr("name", text(label))], + }) + }) + .collect(), + _ => continue, + }; + out.push(json!({ + "name": format!("gitagent.{name}"), + "sum": {"dataPoints": points, "aggregationTemporality": 2, "isMonotonic": true}, + })); + } + } + json!({"resourceMetrics": [{ + "resource": resource(service), + "scopeMetrics": [{"scope": {"name": "gitagent"}, "metrics": out}], + }]}) +} + +fn now_nanos() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos() as u64) +} + +/// A hex id of `bytes` length derived from `seed` (unique enough per run). +fn hex_id(bytes: usize, seed: u64) -> String { + let mut x = seed ^ 0x9e37_79b9_7f4a_7c15 ^ u64::from(std::process::id()); + let mut out = String::new(); + while out.len() < bytes * 2 { + // splitmix64 + x = x.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = x; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + out.push_str(&format!("{:016x}", z ^ (z >> 31))); + } + out.truncate(bytes * 2); + out +} + +/// Parses `YYYY-MM-DDTHH:MM:SS[.fff]Z` into Unix nanoseconds. +pub(crate) fn parse_rfc3339(s: &str) -> Option { + let s = s.strip_suffix('Z')?; + let (date, time) = s.split_once('T')?; + let mut d = date.split('-').map(|p| p.parse::()); + let (y, m, day) = (d.next()?.ok()?, d.next()?.ok()?, d.next()?.ok()?); + let (hms, frac) = time.split_once('.').unwrap_or((time, "0")); + let mut t = hms.split(':').map(|p| p.parse::()); + let (hh, mm, ss) = (t.next()?.ok()?, t.next()?.ok()?, t.next()?.ok()?); + // days_from_civil (Howard Hinnant) + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let mp = (m + 9) % 12; + let doy = (153 * mp + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146_097 + doe - 719_468; + let secs = days * 86_400 + hh * 3600 + mm * 60 + ss; + let nanos: i64 = format!("{frac:0<9}")[..9].parse().ok()?; + u64::try_from(secs * 1_000_000_000 + nanos).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use gitagent_core::Outcome; + use gitagent_provider::Usage; + use gitagent_record::Counters; + + fn report() -> RunReport { + let mut counters = Counters::new(); + counters.add("turns", 2); + counters.inc_labelled("tool_calls", "exec"); + RunReport { + outcome: Outcome::Completed, + turns: 2, + usage: Usage { + input_tokens: 100, + output_tokens: 20, + cache_read_tokens: 60, + cache_write_tokens: 0, + }, + counters, + trajectory_path: "t.json".into(), + summary: None, + } + } + + #[test] + fn parses_our_timestamps() { + assert_eq!(parse_rfc3339("1970-01-01T00:00:00.000Z"), Some(0)); + assert_eq!( + parse_rfc3339("2026-10-14T13:54:56.789Z"), + Some(1_791_986_096_789_000_000) + ); + assert_eq!(parse_rfc3339("garbage"), None); + } + + #[test] + fn builds_spans_and_metrics() { + let t = json!({"steps": [ + {"source": "system", "timestamp": "2026-10-14T10:00:00.000Z"}, + {"source": "user", "timestamp": "2026-10-14T10:00:00.100Z"}, + {"source": "agent", "timestamp": "2026-10-14T10:00:02.000Z", + "metrics": {"prompt_tokens": 50, "completion_tokens": 10, "cached_tokens": 0}, + "tool_calls": [{"function_name": "exec"}]}, + {"source": "agent", "timestamp": "2026-10-14T10:00:03.000Z", + "metrics": {"prompt_tokens": 50, "completion_tokens": 10, "cached_tokens": 60}, + "tool_calls": [{"function_name": "task_complete"}]} + ]}); + let tr = traces( + &t, + &report(), + "anthropic:m", + "svc", + 1_792_000_000_000_000_000, + ); + let spans = tr["resourceSpans"][0]["scopeSpans"][0]["spans"] + .as_array() + .unwrap(); + let names: Vec<&str> = spans.iter().map(|s| s["name"].as_str().unwrap()).collect(); + assert_eq!( + names, + [ + "gitagent.run", + "gitagent.turn", + "gitagent.tool", + "gitagent.turn", + "gitagent.tool" + ] + ); + assert_eq!(spans[1]["parentSpanId"], spans[0]["spanId"]); + assert_eq!(spans[2]["parentSpanId"], spans[1]["spanId"]); + assert_eq!(spans[0]["traceId"].as_str().unwrap().len(), 32); + assert_eq!(spans[1]["startTimeUnixNano"], "1791972000100000000"); + let ids: std::collections::HashSet<&str> = spans + .iter() + .map(|s| s["spanId"].as_str().unwrap()) + .collect(); + assert_eq!(ids.len(), spans.len()); + + let m = metrics(&report(), "svc", 5); + let list = m["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] + .as_array() + .unwrap(); + let tool = list + .iter() + .find(|x| x["name"] == "gitagent.tool_calls") + .unwrap(); + assert_eq!(tool["sum"]["dataPoints"][0]["asInt"], "1"); + assert_eq!( + tool["sum"]["dataPoints"][0]["attributes"][0]["value"]["stringValue"], + "exec" + ); + } +} diff --git a/crates/cli/src/repl.rs b/crates/cli/src/repl.rs new file mode 100644 index 00000000..dffc21d7 --- /dev/null +++ b/crates/cli/src/repl.rs @@ -0,0 +1,66 @@ +//! Interactive mode: tasks are read from stdin one line at a time. Each task +//! continues the same conversation and shell. Ctrl-C stops the current task +//! (and starts a fresh conversation); Ctrl-D or `exit` ends the session. + +use std::io::Write; +use std::path::Path; + +use anyhow::Result; +use gitagent_core::{Resume, RunOptions}; +use tokio::io::{AsyncBufReadExt, BufReader}; + +use crate::{report, Setup}; + +pub(crate) async fn run(s: Setup, out_dir: &Path) -> Result<()> { + eprintln!( + "Type a task and press Enter. Ctrl-C stops the current task; Ctrl-D or `exit` quits." + ); + let mut lines = BufReader::new(tokio::io::stdin()).lines(); + let mut resume: Option = None; + let mut n = 0; + loop { + eprint!("> "); + let _ = std::io::stderr().flush(); + let Some(line) = lines.next_line().await? else { + eprintln!(); + break; + }; + let task = line.trim(); + if task.is_empty() { + continue; + } + if matches!(task, "exit" | "quit") { + break; + } + n += 1; + let opts = RunOptions { + instruction: task.to_string(), + work_dir: s.work_dir.clone(), + out_dir: out_dir.join(format!("task-{n}")), + timeout: Some(s.timeout), + session_id: None, + events: s.events.clone(), + resume: resume.take(), + sandbox: s.sandbox.clone(), + tool_allowlist: None, + tool_denylist: Vec::new(), + hook_callbacks: None, + }; + tokio::select! { + result = gitagent_core::run(&s.spec, s.provider.as_ref(), opts) => match result { + Ok(finished) => { + report(&finished, !s.events.enabled()); + crate::export_telemetry(&finished, &s.spec).await; + crate::learn_cmd::after_run(&s, &finished.report, &s.work_dir, false).await; + resume = Some(finished.into_resume()); + } + Err(e) => eprintln!("gitagent: {e:#}"), + }, + _ = tokio::signal::ctrl_c() => { + // The run, its shell and its history were dropped with it. + eprintln!("\ngitagent: task stopped; the next task starts a fresh conversation"); + } + } + } + Ok(()) +} diff --git a/crates/cli/src/repo_mode.rs b/crates/cli/src/repo_mode.rs new file mode 100644 index 00000000..6907acd9 --- /dev/null +++ b/crates/cli/src/repo_mode.rs @@ -0,0 +1,206 @@ +//! `--repo `: clone, work on a new branch, commit when the task is +//! accepted, push only with `--push`. + +use std::path::PathBuf; +use std::process::ExitCode; + +use anyhow::{Context, Result}; +use gitagent_core::{Outcome, RunOptions}; +use gitagent_repo::{Repo, RepoOptions}; +use serde_json::json; + +use crate::{report, setup, Cli}; + +pub(crate) async fn run(mut cli: Cli, url: String, task: String) -> ExitCode { + match run_inner(&mut cli, url, task).await { + Ok(code) => code, + Err(e) => { + eprintln!("gitagent: {e:#}"); + ExitCode::from(2) + } + } +} + +async fn run_inner(cli: &mut Cli, url: String, task: String) -> Result { + let token = ["GITAGENT_GIT_TOKEN", "GITHUB_TOKEN"] + .iter() + .find_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty())); + let dir = match &cli.dir { + Some(d) => d.clone(), + None => std::env::current_dir()?.join(repo_name(&url)), + }; + // Branch prefix, author and push default come from the user's config. + let home = std::env::var_os("HOME").map(PathBuf::from); + let home_cfg = gitagent_loader::load_config(cli.config.as_deref(), home.as_deref())?.repo; + + let opts = RepoOptions { + url: url.clone(), + base: cli.base.clone(), + branch: cli.branch.clone(), + dir: dir.clone(), + branch_prefix: home_cfg.branch_prefix.clone(), + author: home_cfg.author.clone(), + token, + helper_dir: cli.out_dir.join("askpass"), + }; + let repo = Repo::clone_for_task(&opts, &task)?; + + cli.dir = Some(repo.dir.clone()); + let s = setup(cli)?; + let events = s.events.clone(); + events.emit( + "repo", + json!({"action": "clone", "url": without_credentials(&url), "dir": repo.dir}), + ); + events.emit( + "repo", + json!({"action": "branch", "branch": repo.branch, "base": repo.base, "resumed": repo.resumed}), + ); + eprintln!( + "gitagent: working on branch {} (from {}) in {}", + repo.branch, + repo.base, + repo.dir.display() + ); + + let instruction = format!( + "{}\n\nYou are working in a git clone on branch `{}` (based on `{}`). Your changes are committed for you when you call task_complete; do not commit, push or switch branches yourself.\n\nCurrent `git status`:\n{}", + task.trim(), + repo.branch, + repo.base, + repo.status().unwrap_or_default().trim_end() + ); + let finished = gitagent_core::run( + &s.spec, + s.provider.as_ref(), + RunOptions { + instruction, + work_dir: repo.dir.clone(), + out_dir: cli.out_dir.clone(), + timeout: Some(s.timeout), + session_id: cli.session_id.clone(), + events: events.clone(), + resume: None, + sandbox: s.sandbox.clone(), + tool_allowlist: None, + tool_denylist: Vec::new(), + hook_callbacks: None, + }, + ) + .await?; + report(&finished, !events.enabled()); + crate::export_telemetry(&finished, &s.spec).await; + drop(finished.terminal); + + let mut commit = None; + let mut pushed = false; + if finished.report.outcome == Outcome::Completed { + let subject = finished + .report + .summary + .as_deref() + .unwrap_or(&task) + .lines() + .next() + .unwrap_or("changes") + .to_string(); + let subject = shorten(&subject, 72); + commit = repo + .commit(&format!("gitagent: {subject}")) + .context("committing the changes")?; + events.emit("repo", json!({"action": "commit", "sha": commit})); + // Lessons go in their own commit so they are reviewed with the work. + if crate::learn_cmd::after_run(&s, &finished.report, &repo.dir, false).await { + if let Some(sha) = repo + .commit("gitagent: update learned lessons") + .context("committing the lessons")? + { + events.emit("repo", json!({"action": "commit", "sha": sha})); + commit = Some(sha); + } + } + if (cli.push || home_cfg.push) && (commit.is_some() || repo.resumed) { + repo.push()?; + pushed = true; + events.emit("repo", json!({"action": "push", "branch": repo.branch})); + } + } else { + eprintln!( + "gitagent: the task did not complete ({}); nothing was committed. The work is in {}", + finished.report.outcome, + repo.dir.display() + ); + } + eprintln!( + "gitagent: branch {} · commit {} · {}", + repo.branch, + commit + .as_deref() + .unwrap_or("none (no changes or not completed)"), + if pushed { "pushed" } else { "not pushed" } + ); + Ok( + if matches!(finished.report.outcome, Outcome::ProviderFailed(_)) { + ExitCode::from(3) + } else { + ExitCode::SUCCESS + }, + ) +} + +/// Cuts `text` to at most `max` characters at a word boundary. +fn shorten(text: &str, max: usize) -> String { + if text.chars().count() <= max { + return text.to_string(); + } + let cut: String = text.chars().take(max - 1).collect(); + match cut.rfind(' ') { + Some(i) if i > max / 2 => format!("{}…", cut[..i].trim_end_matches([',', ';', ':'])), + _ => format!("{cut}…"), + } +} + +/// The last path part of the URL without `.git`. +fn repo_name(url: &str) -> String { + let last = url + .trim_end_matches('/') + .rsplit(['/', ':']) + .next() + .unwrap_or("repo"); + let name = last.trim_end_matches(".git"); + if name.is_empty() { + "repo".into() + } else { + name.into() + } +} + +/// Removes `user:password@` from a URL before it is shown or logged. +fn without_credentials(url: &str) -> String { + match (url.find("://"), url.find('@')) { + (Some(scheme), Some(at)) if at > scheme => { + format!("{}{}", &url[..scheme + 3], &url[at + 1..]) + } + _ => url.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_and_credentials() { + assert_eq!(repo_name("https://github.com/org/app.git"), "app"); + assert_eq!(repo_name("git@github.com:org/tool"), "tool"); + assert_eq!(repo_name("/tmp/origin.git/"), "origin"); + assert_eq!( + without_credentials("https://user:tok@github.com/org/app.git"), + "https://github.com/org/app.git" + ); + assert_eq!( + without_credentials("git@github.com:org/app.git"), + "git@github.com:org/app.git" + ); + } +} diff --git a/crates/cli/src/sandbox_exec.rs b/crates/cli/src/sandbox_exec.rs new file mode 100644 index 00000000..2c0915d3 --- /dev/null +++ b/crates/cli/src/sandbox_exec.rs @@ -0,0 +1,50 @@ +//! `gitagent __sandbox --write ... -- [args...]`: applies the +//! sandbox to this process, then replaces it with the program (the agent's +//! shell). Used only internally, when the sandbox is on. + +use std::ffi::OsString; +use std::os::unix::process::CommandExt; +use std::path::PathBuf; +use std::process::{Command, ExitCode}; + +use gitagent_sandbox::{Policy, Status}; + +pub(crate) fn run(args: &[OsString]) -> ExitCode { + let mut writable = Vec::new(); + let mut rest = args.iter(); + let mut program = None; + while let Some(a) = rest.next() { + match a.to_str() { + Some("--write") => { + if let Some(dir) = rest.next() { + writable.push(PathBuf::from(dir)); + } + } + Some("--") => { + program = rest.next().cloned(); + break; + } + _ => { + eprintln!("gitagent __sandbox: unexpected argument {a:?}"); + return ExitCode::from(2); + } + } + } + let Some(program) = program else { + eprintln!("gitagent __sandbox: missing `-- `"); + return ExitCode::from(2); + }; + match gitagent_sandbox::apply(&Policy { writable }) { + Ok(Status::Enforced) => {} + Ok(Status::Partial(note) | Status::Unsupported(note)) => { + eprintln!("gitagent: sandbox limited: {note}"); + } + Err(e) => { + eprintln!("gitagent: sandbox failed: {e:#}"); + return ExitCode::from(2); + } + } + let err = Command::new(&program).args(rest).exec(); + eprintln!("gitagent __sandbox: cannot start {program:?}: {err}"); + ExitCode::from(127) +} diff --git a/crates/cli/tests/cli.rs b/crates/cli/tests/cli.rs new file mode 100644 index 00000000..24bea825 --- /dev/null +++ b/crates/cli/tests/cli.rs @@ -0,0 +1,700 @@ +//! Runs the built binary end to end with a scripted provider. + +use std::process::Command; + +fn gitagent() -> Command { + Command::new(env!("CARGO_BIN_EXE_gitagent")) +} + +#[test] +fn bench_run_writes_trajectory_and_counters() { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("script.yaml"); + std::fs::write( + &script, + "- tool_calls: [{name: exec, arguments: {cmd: 'echo hello > out.txt && cat out.txt'}}]\n\ + - tool_calls: [{name: task_complete, arguments: {summary: wrote out.txt}}]\n", + ) + .unwrap(); + let task = dir.path().join("task.md"); + std::fs::write(&task, "Write hello to out.txt").unwrap(); + let work = dir.path().join("work"); + std::fs::create_dir(&work).unwrap(); + let out = dir.path().join("logs"); + + let status = gitagent() + .arg("--bench") + .arg("--instruction-file") + .arg(&task) + .arg("--model") + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(&work) + .arg("--out-dir") + .arg(&out) + .arg("--timeout-s") + .arg("60") + .status() + .unwrap(); + assert!(status.success()); + assert_eq!( + std::fs::read_to_string(work.join("out.txt")).unwrap(), + "hello\n" + ); + + let t: serde_json::Value = + serde_json::from_slice(&std::fs::read(out.join("trajectory.json")).unwrap()).unwrap(); + assert_eq!(t["schema_version"], "ATIF-v1.8"); + assert_eq!(t["extra"]["outcome"], "completed"); + let c: serde_json::Value = + serde_json::from_slice(&std::fs::read(out.join("counters.json")).unwrap()).unwrap(); + assert_eq!(c["tool_calls"]["exec"], 1); + assert_eq!(std::fs::read_to_string(out.join("done")).unwrap(), "0\n"); +} + +#[test] +fn empty_task_is_a_usage_error() { + let out = gitagent() + .args(["-p", " ", "--model", "scripted:x"]) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("the task is empty")); +} + +#[test] +fn provider_failure_has_its_own_exit_code() { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("empty.yaml"); + std::fs::write(&script, "[]\n").unwrap(); + let status = gitagent() + .args(["-p", "anything", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join("logs")) + .status() + .unwrap(); + assert_eq!(status.code(), Some(3)); + assert_eq!( + std::fs::read_to_string(dir.path().join("logs/done")).unwrap(), + "3\n" + ); +} + +#[test] +fn anthropic_without_a_key_is_a_clear_usage_error() { + let out = gitagent() + .args(["-p", "anything", "--model", "anthropic:claude-opus-5-5"]) + .env_remove("ANTHROPIC_API_KEY") + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("ANTHROPIC_API_KEY is not set")); +} + +#[test] +fn linger_keeps_a_server_alive_after_the_run() { + let dir = tempfile::tempdir().unwrap(); + let work = dir.path().join("work"); + std::fs::create_dir(&work).unwrap(); + let pidfile = work.join("job.pid"); + // A background job with no nohup and no redirection: it dies with its + // shell unless gitagent keeps the shell open. + let script = dir.path().join("script.yaml"); + std::fs::write( + &script, + format!( + "- tool_calls: [{{name: exec, arguments: {{cmd: 'while true; do echo tick; sleep 0.2; done & echo $! > {}'}}}}]\n\ + - tool_calls: [{{name: task_complete}}]\n", + pidfile.display() + ), + ) + .unwrap(); + let out = dir.path().join("logs"); + let mut child = gitagent() + .args(["-p", "x", "--linger", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(&work) + .arg("--out-dir") + .arg(&out) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + let start = std::time::Instant::now(); + while !out.join("done").exists() { + assert!(start.elapsed().as_secs() < 30, "no done file"); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + std::thread::sleep(std::time::Duration::from_millis(500)); + let pid = std::fs::read_to_string(&pidfile) + .unwrap() + .trim() + .to_string(); + let alive = |pid: &str| { + Command::new("kill") + .args(["-0", pid]) + .status() + .unwrap() + .success() + }; + assert!(alive(&pid), "job died although gitagent lingers"); + assert!(child.try_wait().unwrap().is_none(), "gitagent exited"); + child.kill().unwrap(); + child.wait().unwrap(); + let _ = Command::new("kill").arg(&pid).status(); +} + +#[test] +fn openai_without_a_key_is_a_clear_usage_error() { + let out = gitagent() + .args(["-p", "anything", "--model", "openai:some-model"]) + .env_remove("OPENAI_API_KEY") + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("OPENAI_API_KEY is not set")); +} + +#[test] +fn google_and_compat_report_missing_settings() { + let out = gitagent() + .args(["-p", "x", "--model", "google:gemini-x"]) + .env_remove("GEMINI_API_KEY") + .env_remove("GOOGLE_API_KEY") + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("GEMINI_API_KEY is not set")); + + let out = gitagent() + .args(["-p", "x", "--model", "mylab:some-model"]) + .env_remove("MYLAB_BASE_URL") + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("set MYLAB_BASE_URL")); +} + +#[test] +fn project_config_is_used_outside_bench_and_ignored_in_bench() { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("s.yaml"); + std::fs::write(&script, "- tool_calls: [{name: task_complete}]\n").unwrap(); + // A project config that would make every run fail to load. + std::fs::write( + dir.path().join("gitagent.yaml"), + "max_turns: [not a number]\n", + ) + .unwrap(); + let run = |bench: bool| { + let mut cmd = gitagent(); + cmd.args(["-p", "x", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join("logs")) + .env("HOME", dir.path()); + if bench { + cmd.arg("--bench"); + } + cmd.output().unwrap() + }; + let out = run(false); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("gitagent.yaml")); + assert!(run(true).status.success()); +} + +#[test] +fn interactive_session_continues_between_tasks() { + use std::io::Write as _; + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("s.yaml"); + std::fs::write( + &script, + "- tool_calls: [{name: exec, arguments: {cmd: 'export GA_X=7'}}]\n\ + - tool_calls: [{name: task_complete, arguments: {summary: first done}}]\n\ + - tool_calls: [{name: exec, arguments: {cmd: 'echo $GA_X > x.txt'}}]\n\ + - tool_calls: [{name: task_complete, arguments: {summary: second done}}]\n", + ) + .unwrap(); + let mut child = gitagent() + .args(["--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join("logs")) + .env("HOME", dir.path()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(b"set things up\nwrite x\nexit\n") + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + assert_eq!( + String::from_utf8_lossy(&out.stdout), + "first done\nsecond done\n" + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("x.txt")).unwrap(), + "7\n" + ); + assert!(dir.path().join("logs/task-2/trajectory.json").exists()); +} + +#[test] +fn events_go_to_stdout_as_json_lines() { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("s.yaml"); + std::fs::write( + &script, + "- tool_calls: [{name: task_complete, arguments: {summary: ok}}]\n", + ) + .unwrap(); + let out = gitagent() + .args(["-p", "x", "--events", "jsonl", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join("logs")) + .env("HOME", dir.path()) + .output() + .unwrap(); + assert!(out.status.success()); + let lines: Vec = String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + assert_eq!(lines.first().unwrap()["event"], "turn_start"); + assert_eq!(lines.last().unwrap()["event"], "run_end"); + assert_eq!(lines.last().unwrap()["summary"], "ok"); +} + +#[test] +fn bench_without_a_task_is_a_usage_error() { + let out = gitagent() + .args(["--bench", "--model", "scripted:x"]) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("--bench needs a task")); +} + +fn bare_remote(root: &std::path::Path) -> String { + let git = |dir: &std::path::Path, args: &[&str]| { + let out = Command::new("git") + .current_dir(dir) + .args(args) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + }; + git(root, &["init", "-q", "--bare", "-b", "main", "origin.git"]); + git(root, &["init", "-q", "-b", "main", "seed"]); + std::fs::write(root.join("seed/README.md"), "hi\n").unwrap(); + git(&root.join("seed"), &["add", "."]); + git( + &root.join("seed"), + &[ + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "-m", + "init", + ], + ); + let bare = root.join("origin.git").to_string_lossy().into_owned(); + git(&root.join("seed"), &["push", "-q", &bare, "main"]); + bare +} + +#[test] +fn repo_mode_clones_commits_and_pushes_without_leaking_the_token() { + let root = tempfile::tempdir().unwrap(); + let url = bare_remote(root.path()); + let script = root.path().join("s.yaml"); + std::fs::write( + &script, + "- tool_calls: [{name: write_file, arguments: {path: fix.txt, content: \"fixed\\n\"}}]\n\ + - tool_calls: [{name: task_complete, arguments: {summary: Add fix.txt}}]\n", + ) + .unwrap(); + let out = gitagent() + .args([ + "--repo", + &url, + "-p", + "Fix the bug", + "--push", + "--events", + "jsonl", + "--model", + ]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(root.path().join("work")) + .arg("--out-dir") + .arg(root.path().join("logs")) + .env("HOME", root.path()) + .env("GITAGENT_GIT_TOKEN", "supersecret-token-xyz") + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let events: Vec = stdout + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + let repo_events: Vec<&serde_json::Value> = + events.iter().filter(|e| e["event"] == "repo").collect(); + let actions: Vec<&str> = repo_events + .iter() + .map(|e| e["action"].as_str().unwrap()) + .collect(); + assert_eq!(actions, ["clone", "branch", "commit", "push"]); + let branch = repo_events[1]["branch"].as_str().unwrap(); + assert!(branch.starts_with("gitagent/fix-the-bug-")); + + // The remote has the branch with the commit; main is untouched. + let log = Command::new("git") + .args(["--git-dir", &url, "log", "--format=%s", branch]) + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&log.stdout).lines().next(), + Some("gitagent: Add fix.txt") + ); + let main = Command::new("git") + .args(["--git-dir", &url, "log", "--format=%s", "main"]) + .output() + .unwrap(); + assert_eq!(String::from_utf8_lossy(&main.stdout).trim(), "init"); + + // The token appears nowhere: output, events, logs, trajectory, .git. + assert!(!stdout.contains("supersecret")); + assert!(!String::from_utf8_lossy(&out.stderr).contains("supersecret")); + let mut stack = vec![root.path().join("logs"), root.path().join("work/.git")]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let p = entry.path(); + if p.is_dir() { + stack.push(p); + } else if let Ok(bytes) = std::fs::read(&p) { + assert!( + !String::from_utf8_lossy(&bytes).contains("supersecret"), + "{}", + p.display() + ); + } + } + } +} + +#[test] +fn repo_mode_refuses_bench_and_the_base_branch() { + let out = gitagent() + .args(["--bench", "--repo", "x", "-p", "t", "--model", "scripted:x"]) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("--repo cannot be used with --bench")); + + let root = tempfile::tempdir().unwrap(); + let url = bare_remote(root.path()); + let out = gitagent() + .args([ + "--repo", + &url, + "--branch", + "main", + "-p", + "t", + "--model", + "scripted:x", + ]) + .arg("--dir") + .arg(root.path().join("w")) + .env("HOME", root.path()) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("refusing to work on the base branch")); +} + +#[test] +fn writes_under_tests_are_refused_in_every_mode() { + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("s.yaml"); + std::fs::write( + &script, + "- tool_calls: [{name: write_file, arguments: {path: /tests/test_x.py, content: x}}]\n\ + - tool_calls: [{name: task_complete}]\n", + ) + .unwrap(); + let out = gitagent() + .args(["--bench", "-p", "x", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join("logs")) + .output() + .unwrap(); + assert!(out.status.success()); + let c: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("logs/counters.json")).unwrap()) + .unwrap(); + assert_eq!(c["protected_writes_blocked"], 1); +} + +#[test] +fn telemetry_is_posted_when_an_endpoint_is_set() { + use std::io::{Read, Write as _}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let mut paths = Vec::new(); + for _ in 0..2 { + let (mut s, _) = listener.accept().unwrap(); + let mut buf = Vec::new(); + let mut chunk = [0u8; 65536]; + loop { + let n = s.read(&mut chunk).unwrap(); + buf.extend_from_slice(&chunk[..n]); + let text = String::from_utf8_lossy(&buf).to_string(); + if let Some(h) = text.find("\r\n\r\n") { + let len: usize = text[..h] + .lines() + .find_map(|l| { + l.to_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().parse().unwrap()) + }) + .unwrap_or(0); + if buf.len() >= h + 4 + len { + break; + } + } + if n == 0 { + break; + } + } + let text = String::from_utf8_lossy(&buf).to_string(); + paths.push(( + text.lines().next().unwrap().to_string(), + text.contains("x-team: agents"), + )); + s.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}") + .unwrap(); + } + paths + }); + let dir = tempfile::tempdir().unwrap(); + let script = dir.path().join("s.yaml"); + std::fs::write(&script, "- tool_calls: [{name: task_complete}]\n").unwrap(); + let out = gitagent() + .args(["-p", "x", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join("logs")) + .env("HOME", dir.path()) + .env("OTEL_EXPORTER_OTLP_ENDPOINT", format!("http://{addr}")) + .env("OTEL_EXPORTER_OTLP_HEADERS", "x-team=agents") + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let paths = server.join().unwrap(); + assert_eq!(paths[0].0, "POST /v1/traces HTTP/1.1"); + assert_eq!(paths[1].0, "POST /v1/metrics HTTP/1.1"); + assert!(paths[0].1 && paths[1].1); +} + +#[test] +fn online_learning_updates_project_lessons_but_never_in_bench() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("gitagent.yaml"), "learning: online\n").unwrap(); + let lesson = "Check that a command really succeeded before relying on its result."; + let run = |name: &str, learner: &str, bench: bool| { + let script = dir.path().join(format!("{name}.yaml")); + std::fs::write( + &script, + format!( + "- tool_calls: [{{name: task_complete, arguments: {{summary: done}}}}]\n- text: {}\n", + serde_json::to_string(learner).unwrap() + ), + ) + .unwrap(); + let mut cmd = gitagent(); + cmd.args(["-p", "Say hello", "--model"]) + .arg(format!("scripted:{}", script.display())) + .arg("--dir") + .arg(dir.path()) + .arg("--out-dir") + .arg(dir.path().join(format!("logs-{name}"))) + .env("HOME", dir.path()); + if bench { + cmd.arg("--bench"); + } + let out = cmd.output().unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stderr).into_owned() + }; + let store = || -> serde_json::Value { + serde_json::from_slice(&std::fs::read(dir.path().join(".gitagent/lessons.json")).unwrap()) + .unwrap() + }; + let shown = + || std::fs::read_to_string(dir.path().join(".gitagent/prompts/lessons.md")).unwrap(); + + let err = run( + "a", + &format!(r#"{{"ops": [{{"op": "add", "text": "{lesson}"}}]}}"#), + false, + ); + assert!(err.contains("gitagent: learned 1 new"), "{err}"); + assert_eq!(store()["lessons"][0]["text"], lesson); + // One run is not enough evidence to show it to the agent. + assert!(!shown().contains(lesson)); + assert!(shown().starts_with("")) { + Some((_, rest)) => rest.trim(), + None => text.trim(), + } +} + +/// The user message: the current lessons, then each attempt. +pub fn request_text(store: &Store, attempts: &[&Trial]) -> String { + let mut out = String::from("# Current lessons\n\n"); + if store.lessons.is_empty() { + out.push_str("(none yet)\n"); + } + for l in &store.lessons { + out.push_str(&format!("[{}] {}\n", l.id, l.text)); + } + if attempts.len() > 1 { + out.push_str("\nThese are attempts at the same task.\n"); + } + for (i, t) in attempts.iter().enumerate() { + let verdict = match t.reward { + Some(_) if t.passed() => "accepted by the checker", + Some(_) => "rejected by the checker", + None => "not checked by anyone: judge it only from what the trajectory shows", + }; + out.push_str(&format!( + "\n# Attempt {} ({verdict})\n\n{}\n", + (b'A' + i as u8) as char, + t.trace.trim_end() + )); + } + out +} + +pub async fn propose( + spec: &AgentSpec, + provider: &dyn Provider, + store: &Store, + attempts: &[&Trial], +) -> Result> { + let req = ChatRequest { + model: spec.model.model.clone(), + system: learner_prompt().to_string(), + messages: vec![Message::user_text(request_text(store, attempts))], + tools: Vec::new(), + effort: spec.profile.effort, + temperature: spec.profile.temperature, + max_output_tokens: spec.profile.max_output_tokens, + cache: CachePlan::default(), + server_masking: None, + }; + let mut attempt = 0; + let resp = loop { + attempt += 1; + match provider.complete(&req).await { + Ok(r) => break r, + Err(ProviderError::Transient(e)) if attempt < RETRIES => { + eprintln!("gitagent learn: retrying after a provider error: {e}"); + tokio::time::sleep(Duration::from_secs(2u64.pow(attempt))).await; + } + Err(e) => return Err(e.into()), + } + }; + parse_ops(&resp.text) +} + +#[derive(Deserialize)] +struct Reply { + ops: Vec, +} + +/// The `{"ops": [...]}` object in the reply, allowing text around it. +pub fn parse_ops(text: &str) -> Result> { + let (Some(start), Some(end)) = (text.find('{'), text.rfind('}')) else { + bail!("the learner's reply has no JSON object"); + }; + if end < start { + bail!("the learner's reply has no JSON object"); + } + let reply: Reply = serde_json::from_str(&text[start..=end]) + .map_err(|e| anyhow::anyhow!("the learner's reply is not valid: {e}"))?; + Ok(reply.ops) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_ops_with_text_around_them() { + let ops = parse_ops( + "Here you go:\n```json\n{\"ops\": [{\"op\": \"add\", \"text\": \"x y\"}, {\"op\": \"upvote\", \"id\": 3}]}\n```", + ) + .unwrap(); + assert_eq!( + ops, + vec![Op::Add { text: "x y".into() }, Op::Upvote { id: 3 }] + ); + assert_eq!(parse_ops("{\"ops\": []}").unwrap(), vec![]); + assert!(parse_ops("nothing").is_err()); + assert!(parse_ops("{\"ops\": [{\"op\": \"delete\", \"id\": 1}]}").is_err()); + } + + #[test] + fn the_prompt_has_no_header_comment() { + assert!(learner_prompt().starts_with("You keep a short list")); + assert!(!learner_prompt().contains("-->")); + } +} diff --git a/crates/learn/src/lib.rs b/crates/learn/src/lib.rs new file mode 100644 index 00000000..cc7def9e --- /dev/null +++ b/crates/learn/src/lib.rs @@ -0,0 +1,292 @@ +//! Continuous learning: distils general lessons from gitagent's own finished +//! trajectories and their pass/fail outcomes into `prompts/lessons.md`. +//! +//! The learner reads only each trial's `agent/trajectory.json` and the reward +//! in its `result.json`; it never reads tests, verifier output or solutions. With a +//! split file it learns from dev tasks only. Every proposed lesson passes a +//! leakage audit before it is stored, and the rendered file is reviewed and +//! committed like any other prompt change. + +mod audit; +mod distill; +mod store; +mod trace; + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use gitagent_loader::AgentSpec; +use gitagent_provider::Provider; +use serde::Serialize; + +pub use audit::check as audit_lesson; +pub use store::{Applied, Lesson, Op, Store}; +pub use trace::{read_job, read_trial, Trial}; + +/// Most operations taken from one reply. +const MAX_OPS_PER_CALL: usize = 3; + +pub struct LearnOptions { + /// Harbor job directories to learn from. + pub jobs: Vec, + /// The lesson store (lessons with importance counts). + pub store_path: PathBuf, + /// The rendered lessons file the agent loads. + pub lessons_path: PathBuf, + /// Dev task names; trials of any other task are skipped. `None` learns + /// from every trial (only when no split exists yet). + pub dev_tasks: Option>, + /// Size limit of the rendered lessons. + pub max_chars: usize, + /// Lessons are rendered only once attempts at this many different tasks + /// added or upvoted them. + pub min_tasks: usize, + /// Propose and audit, but write nothing. + pub dry_run: bool, +} + +/// What a learning pass did (also its counters). +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +pub struct LearnReport { + pub trials_read: usize, + pub trials_not_dev: usize, + pub trials_already_learned: usize, + pub trials_learned: usize, + pub model_calls: usize, + pub model_errors: usize, + pub ops_proposed: usize, + pub ops_rejected: usize, + pub lessons_added: usize, + pub lessons_edited: usize, + pub lessons_upvoted: usize, + pub lessons_downvoted: usize, + pub lessons_deleted: usize, + pub duplicates: usize, + pub unknown_ids: usize, + pub lessons_total: usize, + /// Lessons with enough evidence to be in the lessons file. + pub lessons_shown: usize, + /// The rendered lessons (written to the lessons file unless dry-run). + #[serde(skip)] + pub rendered: String, +} + +/// Reads the dev task names from a split file (`{"dev": [...], ...}`). +pub fn read_split(path: &Path) -> Result> { + let text = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + let v: serde_json::Value = + serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display()))?; + let dev = v["dev"] + .as_array() + .with_context(|| format!("{} has no `dev` list", path.display()))?; + Ok(dev + .iter() + .filter_map(|t| t.as_str()) + .map(|t| t.rsplit('/').next().unwrap_or(t).to_string()) + .collect()) +} + +pub async fn learn( + spec: &AgentSpec, + provider: &dyn Provider, + opts: &LearnOptions, +) -> Result { + let mut store = Store::load(&opts.store_path)?; + let mut report = LearnReport::default(); + + let mut by_task: BTreeMap> = BTreeMap::new(); + for job in &opts.jobs { + for trial in read_job(job)? { + report.trials_read += 1; + if opts + .dev_tasks + .as_ref() + .is_some_and(|dev| !dev.contains(&trial.task)) + { + report.trials_not_dev += 1; + continue; + } + if store.learned.contains(&trial.id) { + report.trials_already_learned += 1; + continue; + } + by_task.entry(trial.task.clone()).or_default().push(trial); + } + } + + for trials in by_task.values() { + for unit in units(trials) { + learn_unit(spec, provider, &mut store, &unit, &mut report).await; + } + } + report.lessons_total = store.lessons.len(); + report.rendered = store.render(opts.max_chars, opts.min_tasks); + report.lessons_shown = report.rendered.lines().count(); + + if !opts.dry_run { + store.save(&opts.store_path)?; + write_lessons(&opts.lessons_path, &report.rendered)?; + } + Ok(report) +} + +/// Asks the model about one group of attempts, audits its operations and +/// applies the rest. On a model error the attempts are left unlearned. +async fn learn_unit( + spec: &AgentSpec, + provider: &dyn Provider, + store: &mut Store, + unit: &[&Trial], + report: &mut LearnReport, +) { + report.model_calls += 1; + let ops = match distill::propose(spec, provider, store, unit).await { + Ok(ops) => ops, + Err(e) => { + report.model_errors += 1; + eprintln!("gitagent learn: {}: {e:#}", ids(unit)); + return; + } + }; + report.ops_proposed += ops.len(); + let mut kept = Vec::new(); + for op in ops.into_iter().take(MAX_OPS_PER_CALL) { + let text = match &op { + Op::Add { text } | Op::Edit { text, .. } => Some(text.as_str()), + _ => None, + }; + let rejected = text.and_then(|text| { + unit.iter() + .find_map(|t| audit::check(text, &t.task, &t.instruction)) + .map(|why| (text, why)) + }); + match rejected { + Some((text, why)) => { + report.ops_rejected += 1; + eprintln!("gitagent learn: rejected {text:?}: {why}"); + } + None => kept.push(op), + } + } + let a = store.apply(&kept, &unit[0].task, &ids(unit)); + report.lessons_added += a.added; + report.lessons_edited += a.edited; + report.lessons_upvoted += a.upvoted; + report.lessons_downvoted += a.downvoted; + report.lessons_deleted += a.deleted; + report.duplicates += a.duplicates; + report.unknown_ids += a.unknown_ids; + for t in unit { + store.learned.insert(t.id.clone()); + report.trials_learned += 1; + } +} + +/// Header of a project's lessons file when online learning creates it. +const PROJECT_LESSONS_HEADER: &str = ""; + +/// Online learning for everyday use (never in `--bench`): learns from one +/// finished run into `/.gitagent/lessons.json` and renders +/// `/.gitagent/prompts/lessons.md`, which later runs load on top of +/// the built-in lessons. Each run counts as its own task for the evidence +/// threshold, and nothing checked the result, so the learner is told so. +pub async fn learn_from_run( + spec: &AgentSpec, + provider: &dyn Provider, + trajectory_path: &Path, + project_dir: &Path, +) -> Result { + let text = std::fs::read_to_string(trajectory_path) + .with_context(|| format!("reading {}", trajectory_path.display()))?; + let trajectory: gitagent_record::atif::Trajectory = serde_json::from_str(&text) + .with_context(|| format!("parsing {}", trajectory_path.display()))?; + let (instruction, trace) = trace::render(&trajectory); + let id = format!( + "{}@{}", + trajectory.session_id.as_deref().unwrap_or("run"), + gitagent_record::utc_now_rfc3339() + ); + let trial = Trial { + id: id.clone(), + task: id, + reward: None, + instruction, + trace, + }; + + let dir = project_dir.join(".gitagent"); + let store_path = dir.join("lessons.json"); + let lessons_path = dir.join("prompts/lessons.md"); + let mut store = Store::load(&store_path)?; + let mut report = LearnReport { + trials_read: 1, + ..Default::default() + }; + learn_unit(spec, provider, &mut store, &[&trial], &mut report).await; + report.lessons_total = store.lessons.len(); + report.rendered = store.render(PROJECT_MAX_CHARS, 2); + report.lessons_shown = report.rendered.lines().count(); + store.save(&store_path)?; + if !lessons_path.exists() { + if let Some(d) = lessons_path.parent() { + std::fs::create_dir_all(d)?; + } + std::fs::write(&lessons_path, format!("{PROJECT_LESSONS_HEADER}\n"))?; + } + write_lessons(&lessons_path, &report.rendered)?; + Ok(report) +} + +/// Size limit of a project's lessons file. +const PROJECT_MAX_CHARS: usize = 3000; + +/// What to show the model at once: an accepted and a rejected attempt at the +/// same task together (so it can learn from the difference), the rest alone. +fn units(trials: &[Trial]) -> Vec> { + let passed: Vec<&Trial> = trials.iter().filter(|t| t.passed()).collect(); + let failed: Vec<&Trial> = trials.iter().filter(|t| !t.passed()).collect(); + let mut out = Vec::new(); + let pairs = passed.len().min(failed.len()).min(1); + for i in 0..pairs { + out.push(vec![passed[i], failed[i]]); + } + out.extend(passed[pairs..].iter().map(|t| vec![*t])); + out.extend(failed[pairs..].iter().map(|t| vec![*t])); + out +} + +fn ids(unit: &[&Trial]) -> String { + unit.iter() + .map(|t| t.id.as_str()) + .collect::>() + .join(",") +} + +/// Rewrites the lessons file, keeping its header comment. +pub fn write_lessons(path: &Path, lessons: &str) -> Result<()> { + let existing = std::fs::read_to_string(path).unwrap_or_default(); + let trimmed = existing.trim_start(); + let header = match trimmed.strip_prefix("")) { + Some(end) => &trimmed[..end + "".len()], + None => "", + }; + let mut out = String::new(); + if !header.is_empty() { + out.push_str(header); + out.push('\n'); + } + out.push_str(lessons); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + std::fs::write(path, out).with_context(|| format!("writing {}", path.display())) +} + +#[cfg(test)] +mod tests; diff --git a/crates/learn/src/store.rs b/crates/learn/src/store.rs new file mode 100644 index 00000000..314b03b8 --- /dev/null +++ b/crates/learn/src/store.rs @@ -0,0 +1,301 @@ +//! The lesson store: general lessons with per-task importance counts. + +use std::collections::BTreeSet; +use std::path::Path; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +/// Importance of a new lesson; it is deleted when it reaches 0. +const NEW_IMPORTANCE: i32 = 2; +/// Longest lesson kept. +pub const MAX_LESSON_CHARS: usize = 400; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Lesson { + pub id: u32, + pub text: String, + pub importance: i32, + pub helpful: u32, + pub harmful: u32, + /// Tasks whose attempts added or upvoted it (bookkeeping for the + /// evidence threshold; never shown to the model). + #[serde(default)] + pub tasks: BTreeSet, + /// Trials it came from or was voted on by (for review; never shown to the model). + #[serde(default)] + pub sources: Vec, +} + +impl Lesson { + fn vote_from(&mut self, task: &str, source: &str) { + if self.tasks.insert(task.to_string()) { + self.importance += 1; + } + self.sources.push(source.to_string()); + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct Store { + pub next_id: u32, + pub lessons: Vec, + /// Trials already learned from, so re-running on the same jobs is a no-op. + #[serde(default)] + pub learned: BTreeSet, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(tag = "op", rename_all = "lowercase")] +pub enum Op { + Add { text: String }, + Edit { id: u32, text: String }, + Upvote { id: u32 }, + Downvote { id: u32 }, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Applied { + pub added: usize, + pub edited: usize, + pub upvoted: usize, + pub downvoted: usize, + pub deleted: usize, + pub duplicates: usize, + pub unknown_ids: usize, +} + +impl Store { + pub fn load(path: &Path) -> Result { + if !path.exists() { + return Ok(Store::default()); + } + let text = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display())) + } + + pub fn save(&self, path: &Path) -> Result<()> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + std::fs::write(path, serde_json::to_string_pretty(self)? + "\n") + .with_context(|| format!("writing {}", path.display())) + } + + /// Applies operations that already passed the audit. `task` is the task + /// the attempts were for: importance rises only on the first vote from + /// each task, so repeated runs of one task do not count as wider evidence. + pub fn apply(&mut self, ops: &[Op], task: &str, source: &str) -> Applied { + let mut a = Applied::default(); + for op in ops { + match op { + Op::Add { text } => { + let text = clean(text); + if self.lessons.iter().any(|l| similar(&l.text, &text)) { + a.duplicates += 1; + continue; + } + self.next_id += 1; + self.lessons.push(Lesson { + id: self.next_id, + text, + importance: NEW_IMPORTANCE, + helpful: 0, + harmful: 0, + tasks: BTreeSet::from([task.to_string()]), + sources: vec![source.to_string()], + }); + a.added += 1; + } + Op::Edit { id, text } => match self.get(*id) { + Some(l) => { + l.text = clean(text); + l.vote_from(task, source); + a.edited += 1; + } + None => a.unknown_ids += 1, + }, + Op::Upvote { id } => match self.get(*id) { + Some(l) => { + l.helpful += 1; + l.vote_from(task, source); + a.upvoted += 1; + } + None => a.unknown_ids += 1, + }, + Op::Downvote { id } => match self.get(*id) { + Some(l) => { + l.importance -= 1; + l.harmful += 1; + l.sources.push(source.to_string()); + a.downvoted += 1; + } + None => a.unknown_ids += 1, + }, + } + } + let before = self.lessons.len(); + self.lessons.retain(|l| l.importance > 0); + a.deleted = before - self.lessons.len(); + a + } + + fn get(&mut self, id: u32) -> Option<&mut Lesson> { + self.lessons.iter_mut().find(|l| l.id == id) + } + + /// The lessons file: lessons backed by at least `min_tasks` different + /// tasks, most important first, within `max_chars`. + pub fn render(&self, max_chars: usize, min_tasks: usize) -> String { + let mut sorted: Vec<&Lesson> = self + .lessons + .iter() + .filter(|l| l.tasks.len() >= min_tasks) + .collect(); + sorted.sort_by(|a, b| b.importance.cmp(&a.importance).then(a.id.cmp(&b.id))); + let mut out = String::new(); + for l in sorted { + let line = format!("- {}\n", l.text); + if out.len() + line.len() > max_chars { + break; + } + out.push_str(&line); + } + out + } +} + +fn clean(text: &str) -> String { + let one_line = text.split_whitespace().collect::>().join(" "); + one_line.chars().take(MAX_LESSON_CHARS).collect() +} + +/// Same words for the most part (Jaccard similarity of lowercase words). +fn similar(a: &str, b: &str) -> bool { + let words = |s: &str| { + s.to_lowercase() + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| w.len() > 2) + .map(str::to_string) + .collect::>() + }; + let (x, y) = (words(a), words(b)); + if x.is_empty() || y.is_empty() { + return false; + } + let inter = x.intersection(&y).count() as f64; + let union = x.union(&y).count() as f64; + inter / union >= 0.8 +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOHUP: &str = "Start servers with nohup so they outlive the command."; + + #[test] + fn importance_counts_once_per_task() { + let mut s = Store::default(); + let a = s.apply( + &[ + Op::Add { text: NOHUP.into() }, + Op::Add { + text: "Check installed tool versions before writing code.".into(), + }, + Op::Add { + text: "start servers with nohup, so they outlive the command".into(), + }, + ], + "t1", + "t1/a", + ); + assert_eq!((a.added, a.duplicates), (2, 1)); + assert_eq!(s.lessons[0].importance, 2); + + let a = s.apply( + &[ + Op::Upvote { id: 1 }, + Op::Downvote { id: 2 }, + Op::Downvote { id: 9 }, + ], + "t2", + "t2/a", + ); + assert_eq!((a.upvoted, a.downvoted, a.unknown_ids), (1, 1, 1)); + assert_eq!(s.lessons[0].importance, 3); + // A second downvote takes lesson 2 to 0: it is deleted. + let a = s.apply(&[Op::Downvote { id: 2 }], "t3", "t3/a"); + assert_eq!(a.deleted, 1); + assert_eq!(s.lessons.len(), 1); + + let a = s.apply( + &[Op::Edit { + id: 1, + text: "Start long-running servers with nohup.".into(), + }], + "t4", + "t4/a", + ); + assert_eq!(a.edited, 1); + assert_eq!(s.lessons[0].importance, 4); + assert_eq!( + s.render(1000, 1), + "- Start long-running servers with nohup.\n" + ); + } + + #[test] + fn repeated_runs_of_one_task_count_once() { + let mut s = Store::default(); + s.apply(&[Op::Add { text: NOHUP.into() }], "t1", "t1/a"); + s.apply(&[Op::Upvote { id: 1 }], "t1", "t1/b"); + s.apply(&[Op::Upvote { id: 1 }], "t1", "t1/c"); + let l = &s.lessons[0]; + assert_eq!((l.importance, l.helpful, l.sources.len()), (2, 2, 3)); + // One task is not enough evidence to show it to the agent. + assert_eq!(s.render(1000, 2), ""); + s.apply(&[Op::Upvote { id: 1 }], "t2", "t2/a"); + assert_eq!(s.lessons[0].importance, 3); + assert_eq!(s.render(1000, 2), format!("- {NOHUP}\n")); + } + + #[test] + fn render_respects_the_size_limit_and_order() { + let mut s = Store::default(); + s.apply( + &[ + Op::Add { text: "low".into() }, + Op::Add { + text: "high priority lesson".into(), + }, + ], + "t", + "t/a", + ); + s.apply(&[Op::Upvote { id: 2 }], "u", "u/a"); + assert_eq!(s.render(1000, 1), "- high priority lesson\n- low\n"); + assert_eq!(s.render(25, 1), "- high priority lesson\n"); + } + + #[test] + fn saves_and_loads() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("x/lessons.json"); + let mut s = Store::default(); + s.apply( + &[Op::Add { + text: "a lesson".into(), + }], + "t", + "t/a", + ); + s.save(&path).unwrap(); + assert_eq!(Store::load(&path).unwrap(), s); + assert_eq!( + Store::load(&dir.path().join("none.json")).unwrap(), + Store::default() + ); + } +} diff --git a/crates/learn/src/tests.rs b/crates/learn/src/tests.rs new file mode 100644 index 00000000..31eac72b --- /dev/null +++ b/crates/learn/src/tests.rs @@ -0,0 +1,252 @@ +use std::path::Path; + +use gitagent_loader::{load, AgentSpec, LoadOptions}; +use gitagent_provider::ScriptedProvider; + +use super::*; + +pub(crate) fn write_trial(job: &Path, name: &str, task: &str, reward: Option) { + let dir = job.join(name); + std::fs::create_dir_all(dir.join("agent")).unwrap(); + let mut result = serde_json::json!({ "task_name": format!("terminal-bench/{task}") }); + if let Some(r) = reward { + result["verifier_result"] = serde_json::json!({ "rewards": { "reward": r } }); + } + std::fs::write(dir.join("result.json"), result.to_string()).unwrap(); + let trajectory = serde_json::json!({ + "schema_version": "ATIF-v1.8", + "session_id": "s", + "agent": { "name": "gitagent", "version": "0" }, + "steps": [ + { "step_id": 1, "source": "system", "message": "SYSTEM PROMPT" }, + { "step_id": 2, "source": "user", "message": format!("# Task\n\nStart the {task} server on port 5328 from /app/server.py.") }, + { "step_id": 3, "source": "agent", "message": "Starting it.", + "tool_calls": [{ "tool_call_id": "c1", "function_name": "exec", "arguments": { "cmd": "python /app/server.py" } }], + "observation": { "results": [{ "source_call_id": "c1", "content": "x".repeat(5000) }] } } + ] + }); + std::fs::write(dir.join("agent/trajectory.json"), trajectory.to_string()).unwrap(); + // Files the learner must never read. + let never_read = dir.join("verifier"); + std::fs::create_dir_all(&never_read).unwrap(); + std::fs::write(never_read.join("test-stdout.txt"), "SECRET").unwrap(); +} + +fn spec() -> AgentSpec { + load(&LoadOptions { + model: Some("scripted:test".into()), + ..Default::default() + }) + .unwrap() +} + +fn reply(json: &str) -> String { + format!("- text: {}\n", serde_json::to_string(json).unwrap()) +} + +struct Setup { + dir: tempfile::TempDir, + opts: LearnOptions, +} + +/// Two jobs: `serve-thing` passed once and failed once, `other-thing` +/// passed, `heldout-thing` is not in the dev split. +fn setup() -> Setup { + let dir = tempfile::tempdir().unwrap(); + let (j1, j2) = (dir.path().join("job1"), dir.path().join("job2")); + write_trial(&j1, "serve-thing__a", "serve-thing", Some(1.0)); + write_trial(&j1, "other-thing__a", "other-thing", Some(1.0)); + write_trial(&j2, "serve-thing__b", "serve-thing", Some(0.0)); + write_trial(&j2, "heldout-thing__a", "heldout-thing", Some(0.0)); + let lessons_path = dir.path().join("prompts/lessons.md"); + std::fs::create_dir_all(lessons_path.parent().unwrap()).unwrap(); + std::fs::write(&lessons_path, "\n").unwrap(); + let opts = LearnOptions { + jobs: vec![j1, j2], + store_path: dir.path().join("lessons/lessons.json"), + lessons_path, + dev_tasks: Some(["serve-thing", "other-thing"].map(String::from).into()), + max_chars: 3000, + min_tasks: 2, + dry_run: false, + }; + Setup { dir, opts } +} + +const NOHUP: &str = "Start long-running servers with nohup and check they answer before finishing."; +const VERSIONS: &str = "Read the installed library's version before writing code against its API."; + +#[tokio::test] +async fn learns_audits_and_renders_lessons_from_dev_trials_only() { + let s = setup(); + let script = reply(&format!( + r#"{{"ops": [{{"op": "add", "text": "{NOHUP}"}}, {{"op": "add", "text": "Put the code in /app/server.py so it is found."}}]}}"# + )) + &reply(&format!( + r#"{{"ops": [{{"op": "upvote", "id": 1}}, {{"op": "add", "text": "{VERSIONS}"}}]}}"# + )); + let provider = ScriptedProvider::from_yaml(&script).unwrap(); + let mut report = learn(&spec(), &provider, &s.opts).await.unwrap(); + // VERSIONS comes from one task only, so it is stored but not shown yet. + assert_eq!(report.rendered, format!("- {NOHUP}\n")); + report.rendered.clear(); + assert_eq!( + report, + LearnReport { + trials_read: 4, + trials_not_dev: 1, + trials_learned: 3, + model_calls: 2, + ops_proposed: 4, + ops_rejected: 1, + lessons_added: 2, + lessons_upvoted: 1, + lessons_total: 2, + lessons_shown: 1, + ..Default::default() + } + ); + + let requests = provider.requests(); + assert!(requests[0].system.starts_with("You keep a short list")); + assert!(requests[0].tools.is_empty()); + let first = serde_json::to_string(&requests[0].messages).unwrap(); + assert!(first.contains("(none yet)")); + assert!(first.contains("Attempt A (accepted by the checker)")); + // The pass/fail pair of one task goes in one request, after the first + // request's lesson was stored. + let second = serde_json::to_string(&requests[1].messages).unwrap(); + assert!(second.contains(&format!("[1] {NOHUP}"))); + assert!(second.contains("same task")); + assert!(second.contains("Attempt B (rejected by the checker)")); + for r in &requests { + let text = serde_json::to_string(&r.messages).unwrap(); + assert!(!text.contains("SECRET"), "verifier files are never read"); + assert!(!text.contains("heldout-thing")); + assert!(!text.contains("SYSTEM PROMPT")); + } + + let rendered = std::fs::read_to_string(&s.opts.lessons_path).unwrap(); + assert_eq!(rendered, format!("\n- {NOHUP}\n")); + let store = Store::load(&s.opts.store_path).unwrap(); + assert_eq!(store.lessons[0].importance, 3); + assert_eq!(store.learned.len(), 3); + + // A second pass over the same jobs asks nothing. + let provider = ScriptedProvider::from_yaml("[]").unwrap(); + let report = learn(&spec(), &provider, &s.opts).await.unwrap(); + assert_eq!((report.trials_already_learned, report.model_calls), (3, 0)); + assert!(provider.requests().is_empty()); + drop(s.dir); +} + +#[tokio::test] +async fn a_failed_call_leaves_the_trials_for_next_time() { + let mut s = setup(); + s.opts.dev_tasks = Some(["other-thing".to_string()].into()); + let provider = ScriptedProvider::from_yaml("- error: fatal\n").unwrap(); + let report = learn(&spec(), &provider, &s.opts).await.unwrap(); + assert_eq!( + ( + report.model_calls, + report.model_errors, + report.trials_learned + ), + (1, 1, 0) + ); + assert!(Store::load(&s.opts.store_path).unwrap().learned.is_empty()); +} + +#[tokio::test] +async fn dry_run_writes_nothing() { + let mut s = setup(); + s.opts.dry_run = true; + s.opts.min_tasks = 1; + s.opts.dev_tasks = Some(["other-thing".to_string()].into()); + let provider = ScriptedProvider::from_yaml(&reply(&format!( + r#"{{"ops": [{{"op": "add", "text": "{NOHUP}"}}]}}"# + ))) + .unwrap(); + let report = learn(&spec(), &provider, &s.opts).await.unwrap(); + assert_eq!(report.lessons_added, 1); + assert_eq!(report.rendered, format!("- {NOHUP}\n")); + assert!(!s.opts.store_path.exists()); + assert_eq!( + std::fs::read_to_string(&s.opts.lessons_path).unwrap(), + "\n" + ); +} + +#[test] +fn split_files_list_dev_tasks() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("split.json"); + std::fs::write( + &path, + r#"{"dev": ["terminal-bench/repair-repo", "grpc-kv-demo"], "heldout": ["x"]}"#, + ) + .unwrap(); + let dev = read_split(&path).unwrap(); + assert_eq!( + dev, + ["repair-repo", "grpc-kv-demo"].map(String::from).into() + ); + std::fs::write(&path, r#"{"heldout": []}"#).unwrap(); + assert!(read_split(&path).is_err()); +} + +#[test] +fn the_shipped_lessons_file_keeps_its_header() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lessons.md"); + write_lessons(&path, "- a\n").unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "- a\n"); + std::fs::write(&path, "\n- old\n").unwrap(); + write_lessons(&path, "- new\n").unwrap(); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "\n- new\n" + ); +} + +#[tokio::test] +async fn online_learning_writes_into_the_project() { + let dir = tempfile::tempdir().unwrap(); + let job = dir.path().join("job"); + write_trial(&job, "t__1", "serve-thing", None); + let trajectory = job.join("t__1/agent/trajectory.json"); + let project = dir.path().join("project"); + let provider = ScriptedProvider::from_yaml( + &(reply(&format!( + r#"{{"ops": [{{"op": "add", "text": "{NOHUP}"}}]}}"# + )) + &reply(r#"{"ops": [{"op": "upvote", "id": 1}]}"#)), + ) + .unwrap(); + + let r = learn_from_run(&spec(), &provider, &trajectory, &project) + .await + .unwrap(); + assert_eq!( + ( + r.trials_read, + r.trials_learned, + r.lessons_added, + r.lessons_shown + ), + (1, 1, 1, 0) + ); + let text = serde_json::to_string(&provider.requests()[0].messages).unwrap(); + assert!(text.contains("not checked by anyone")); + let lessons = project.join(".gitagent/prompts/lessons.md"); + assert!(std::fs::read_to_string(&lessons) + .unwrap() + .ends_with("-->\n")); + + // A second run supports it: each run counts as its own task. + let r = learn_from_run(&spec(), &provider, &trajectory, &project) + .await + .unwrap(); + assert_eq!((r.lessons_upvoted, r.lessons_shown), (1, 1)); + assert!(std::fs::read_to_string(&lessons) + .unwrap() + .ends_with(&format!("-->\n- {NOHUP}\n"))); +} diff --git a/crates/learn/src/trace.rs b/crates/learn/src/trace.rs new file mode 100644 index 00000000..99f64cf6 --- /dev/null +++ b/crates/learn/src/trace.rs @@ -0,0 +1,164 @@ +//! Reads finished trials: gitagent's own trajectory and the trial's reward. +//! Nothing else in a trial directory is opened: it never reads verifier output or tests. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use gitagent_record::atif::{Source, Trajectory}; +use serde_json::Value; + +/// Longest rendered trace sent to the model; the middle is cut beyond this. +const MAX_TRACE_CHARS: usize = 60_000; +/// Longest single message, argument or observation in the rendered trace. +const MAX_ITEM_CHARS: usize = 2_000; + +#[derive(Debug, Clone)] +pub struct Trial { + /// `/`, used to skip trials already learned from. + pub id: String, + /// The task name, used only for the split check and the leakage audit; + /// it is never shown to the model. + pub task: String, + /// The trial's reward, when the run got as far as being checked. + pub reward: Option, + /// The task instruction (the first user message). + pub instruction: String, + /// The trajectory as plain text for the model. + pub trace: String, +} + +impl Trial { + pub fn passed(&self) -> bool { + self.reward.is_some_and(|r| r >= 1.0) + } +} + +/// Every trial in a Harbor job directory (sub-directories with a +/// `result.json` and an `agent/trajectory.json`), sorted by name. +pub fn read_job(job_dir: &Path) -> Result> { + let job = job_dir + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let mut dirs: Vec = std::fs::read_dir(job_dir) + .with_context(|| format!("reading {}", job_dir.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.join("result.json").is_file() && p.join("agent/trajectory.json").is_file()) + .collect(); + dirs.sort(); + let mut trials = Vec::new(); + for dir in dirs { + let name = dir.file_name().unwrap_or_default().to_string_lossy(); + trials.push(read_trial(&dir, &format!("{job}/{name}"))?); + } + Ok(trials) +} + +pub fn read_trial(dir: &Path, id: &str) -> Result { + let result: Value = read_json(&dir.join("result.json"))?; + let task = result["task_name"] + .as_str() + .or_else(|| result["config"]["task"]["name"].as_str()) + .unwrap_or_default(); + let task = task.rsplit('/').next().unwrap_or(task).to_string(); + let reward = result["verifier_result"]["rewards"]["reward"].as_f64(); + let trajectory: Trajectory = read_json(&dir.join("agent/trajectory.json"))?; + let (instruction, trace) = render(&trajectory); + Ok(Trial { + id: id.to_string(), + task, + reward, + instruction, + trace, + }) +} + +fn read_json(path: &Path) -> Result { + let text = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display())) +} + +/// The instruction and the trajectory as text: each agent step's message, +/// tool calls and observations, long items cut in the middle. The system +/// prompt is left out (the learner sees the lessons separately). +pub fn render(t: &Trajectory) -> (String, String) { + let mut instruction = String::new(); + let mut out = String::new(); + for step in &t.steps { + match step.source { + Source::System => {} + Source::User => { + if instruction.is_empty() { + instruction = step.message.clone(); + } + out.push_str(&format!( + "## user\n{}\n\n", + cut(&step.message, MAX_ITEM_CHARS) + )); + } + Source::Agent => { + out.push_str(&format!("## agent (step {})\n", step.step_id)); + if !step.message.trim().is_empty() { + out.push_str(&cut(&step.message, MAX_ITEM_CHARS)); + out.push('\n'); + } + for call in step.tool_calls.iter().flatten() { + let args = serde_json::to_string(&call.arguments).unwrap_or_default(); + out.push_str(&format!( + "-> {}({})\n", + call.function_name, + cut(&args, MAX_ITEM_CHARS) + )); + } + for r in step.observation.iter().flat_map(|o| &o.results) { + if let Some(c) = &r.content { + out.push_str(&format!("<- {}\n", cut(c, MAX_ITEM_CHARS))); + } + } + out.push('\n'); + } + } + } + (instruction, cut(&out, MAX_TRACE_CHARS)) +} + +/// Keeps the head and tail of long text. +fn cut(text: &str, max: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= max { + return text.to_string(); + } + let head: String = chars[..max / 2].iter().collect(); + let tail: String = chars[chars.len() - max / 2..].iter().collect(); + format!( + "{head}\n[... {} characters cut ...]\n{tail}", + chars.len() - max + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::write_trial; + + #[test] + fn reads_the_trajectory_and_reward_only() { + let job = tempfile::tempdir().unwrap(); + write_trial(job.path(), "a__1", "serve-thing", Some(1.0)); + write_trial(job.path(), "b__2", "other-thing", None); + std::fs::create_dir_all(job.path().join("not-a-trial")).unwrap(); + let trials = read_job(job.path()).unwrap(); + assert_eq!(trials.len(), 2); + let t = &trials[0]; + assert_eq!(t.task, "serve-thing"); + assert!(t.passed()); + assert!(!trials[1].passed()); + assert!(t.instruction.contains("port 5328")); + assert!(!t.trace.contains("SYSTEM PROMPT")); + assert!(t.trace.contains("-> exec(")); + assert!(t.trace.contains("characters cut")); + assert!(!t.trace.contains("SECRET")); + assert!(t.id.ends_with("/a__1")); + } +} diff --git a/crates/loader/Cargo.toml b/crates/loader/Cargo.toml new file mode 100644 index 00000000..2a6f39c1 --- /dev/null +++ b/crates/loader/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "gitagent-loader" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +serde-saphyr = { version = "1.3.0", default-features = false, features = ["deserialize"] } + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/loader/src/agents.rs b/crates/loader/src/agents.rs new file mode 100644 index 00000000..8d3b9d6c --- /dev/null +++ b/crates/loader/src/agents.rs @@ -0,0 +1,218 @@ +//! Sub-agent definitions: `agents/.md` in a config directory (including +//! `.claude/agents/`, in the `.claude/` format), plus a built-in +//! `general` agent. Frontmatter: `name`, `description`, optional `tools` and +//! `disallowedTools` (a comma-separated string or a YAML list, in `.claude/` +//! or gitagent tool names), `maxTurns` (or `max_turns`), `model` and `effort`; +//! the body is the sub-agent's system prompt. + +use std::path::PathBuf; + +use crate::skills::split_frontmatter; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentDef { + pub name: String, + pub description: String, + /// Tools the sub-agent may use (gitagent names); `None` means the parent's tools. + pub tools: Option>, + /// Tools removed from whatever `tools` allows (gitagent names). + pub disallowed_tools: Vec, + pub max_turns: u32, + /// The agent file's `model` field (an alias such as `sonnet`, a model + /// id, or `inherit`). Stored for the sub-agent features; not applied yet. + pub model: Option, + /// The agent file's `effort` field: `low`, `medium`, `high`, `xhigh` or `max`. + pub effort: Option, + pub instructions: String, +} + +pub const GENERAL: &str = "general"; +const DEFAULT_MAX_TURNS: u32 = 30; + +pub fn general() -> AgentDef { + AgentDef { + name: GENERAL.into(), + description: "A helper for a self-contained piece of work (investigating, building or \ + checking something) whose details you do not need to keep in your own context." + .into(), + tools: None, + disallowed_tools: Vec::new(), + max_turns: DEFAULT_MAX_TURNS, + model: None, + effort: None, + instructions: String::new(), + } +} + +/// A frontmatter field by key: `key: value`, `key: [a, b]`, or a YAML list on +/// the following lines. Lists come back joined with commas. +fn front_field(front: &str, key: &str) -> Option { + let lines: Vec<&str> = front.lines().collect(); + let i = lines.iter().position(|l| { + l.strip_prefix(key) + .is_some_and(|r| r.trim_start().starts_with(':') && !l.starts_with(' ')) + })?; + let value = lines[i] + .split_once(':') + .map(|(_, v)| v.trim()) + .unwrap_or(""); + if !value.is_empty() { + let v = value.trim_matches('"').trim_matches('\''); + let v = v + .strip_prefix('[') + .and_then(|x| x.strip_suffix(']')) + .unwrap_or(v); + return Some(v.to_string()); + } + let items: Vec = lines[i + 1..] + .iter() + .take_while(|l| l.trim_start().starts_with('-')) + .map(|l| { + l.trim_start() + .trim_start_matches('-') + .trim() + .trim_matches('"') + .to_string() + }) + .collect(); + Some(items.join(", ")) +} + +fn tool_list(value: Option) -> Option> { + value.map(|t| { + crate::sdk::normalize_tool_list( + &t.split(',') + .map(|s| s.trim().to_string()) + .collect::>(), + ) + }) +} + +/// The built-in agent plus definitions from the config directories (the +/// first directory wins for a name), sorted by name. +pub fn discover(config_dirs: &[PathBuf]) -> Vec { + let mut agents = vec![general()]; + for dir in config_dirs { + let Ok(entries) = std::fs::read_dir(dir.join("agents")) else { + continue; + }; + let mut files: Vec = entries + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "md")) + .collect(); + files.sort(); + for path in files { + let Ok(text) = std::fs::read_to_string(&path) else { + continue; + }; + let (front, body) = split_frontmatter(&text); + let field = |key: &str| front_field(front, key); + let name = field("name") + .filter(|n| !n.is_empty()) + .or_else(|| path.file_stem().map(|s| s.to_string_lossy().into_owned())) + .unwrap_or_default(); + if name.is_empty() || agents.iter().any(|a| a.name == name && a.name != GENERAL) { + continue; + } + let def = AgentDef { + description: field("description").unwrap_or_default(), + tools: tool_list(field("tools")), + disallowed_tools: tool_list( + field("disallowedTools").or_else(|| field("disallowed_tools")), + ) + .unwrap_or_default(), + max_turns: field("maxTurns") + .or_else(|| field("max_turns")) + .and_then(|m| m.parse().ok()) + .unwrap_or(DEFAULT_MAX_TURNS), + model: field("model").filter(|m| !m.is_empty() && m != "inherit"), + effort: field("effort").filter(|e| !e.is_empty()), + instructions: body.trim().to_string(), + name, + }; + // A file can redefine the built-in agent. + agents.retain(|a| a.name != def.name); + agents.push(def); + } + } + agents.sort_by(|a, b| a.name.cmp(&b.name)); + agents +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn built_in_and_file_agents() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("agents")).unwrap(); + std::fs::write( + dir.path().join("agents/reviewer.md"), + "---\nname: reviewer\ndescription: Reviews a diff.\ntools: exec, read_file\nmax_turns: 8\n---\nOnly read; never edit.\n", + ) + .unwrap(); + let agents = discover(&[dir.path().to_path_buf()]); + assert_eq!(agents.len(), 2); + assert_eq!(agents[0].name, "general"); + let r = &agents[1]; + assert_eq!(r.description, "Reviews a diff."); + assert_eq!( + r.tools.as_deref(), + Some(&["exec".to_string(), "read_file".to_string()][..]) + ); + assert_eq!(r.max_turns, 8); + assert_eq!(r.instructions, "Only read; never edit."); + } + + #[test] + fn claude_agent_sdk_files_are_understood() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("agents")).unwrap(); + std::fs::write( + dir.path().join("agents/code-reviewer.md"), + "---\nname: code-reviewer\ndescription: Reviews code for quality.\ntools:\n - Read\n - Grep\n - Bash\ndisallowedTools: Edit, Write\nmodel: sonnet\nmaxTurns: 12\neffort: low\ncolor: cyan\n---\nYou are a code reviewer.\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("agents/writer.md"), + "---\nname: writer\ndescription: Writes docs.\ntools: [Write, Read]\nmodel: inherit\n---\nWrite.\n", + ) + .unwrap(); + let agents = discover(&[dir.path().to_path_buf()]); + let r = agents.iter().find(|a| a.name == "code-reviewer").unwrap(); + // Grep has no gitagent equivalent and is dropped; Bash is exec. + assert_eq!( + r.tools.as_deref(), + Some( + &[ + "read_file".to_string(), + "image_read".to_string(), + "exec".to_string() + ][..] + ) + ); + assert_eq!( + r.disallowed_tools, + vec!["str_replace", "apply_patch", "write_file"] + ); + assert_eq!( + (r.max_turns, r.model.as_deref(), r.effort.as_deref()), + (12, Some("sonnet"), Some("low")) + ); + assert_eq!(r.instructions, "You are a code reviewer."); + let w = agents.iter().find(|a| a.name == "writer").unwrap(); + assert_eq!( + w.tools.as_deref(), + Some( + &[ + "write_file".to_string(), + "read_file".to_string(), + "image_read".to_string() + ][..] + ) + ); + assert_eq!(w.model, None, "inherit means the parent's model"); + } +} diff --git a/crates/loader/src/config.rs b/crates/loader/src/config.rs new file mode 100644 index 00000000..5f003cd2 --- /dev/null +++ b/crates/loader/src/config.rs @@ -0,0 +1,58 @@ +use serde::Deserialize; + +/// `gitagent.yaml` (BLUEPRINT §6). +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct Config { + pub name: String, + pub model: Option, + pub fallback_models: Vec, + pub max_turns: u32, + pub timeout_s: u64, + pub profile: String, + pub sandbox: String, + pub skills: bool, + pub hooks: bool, + /// `off` or `online`: after each finished task outside `--bench`, learn + /// lessons into the project's `.gitagent/`. + pub learning: String, + pub repo: RepoConfig, +} + +impl Default for Config { + fn default() -> Self { + Config { + name: "gitagent".into(), + model: None, + fallback_models: Vec::new(), + max_turns: 200, + timeout_s: 1800, + profile: "auto".into(), + sandbox: "auto".into(), + skills: true, + hooks: true, + learning: "off".into(), + repo: RepoConfig::default(), + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct RepoConfig { + pub branch_prefix: String, + pub commit_every_turns: u32, + pub author: String, + pub push: bool, +} + +impl Default for RepoConfig { + fn default() -> Self { + RepoConfig { + branch_prefix: "gitagent/".into(), + commit_every_turns: 0, + author: "gitagent ".into(), + push: false, + } + } +} diff --git a/crates/loader/src/lib.rs b/crates/loader/src/lib.rs new file mode 100644 index 00000000..aa2faeb0 --- /dev/null +++ b/crates/loader/src/lib.rs @@ -0,0 +1,682 @@ +//! Loads `gitagent.yaml`, the model profile and the prompts into an [`AgentSpec`]. +//! +//! Every file has a built-in default compiled from the repository, so the +//! static binary runs with no files next to it. A config directory given with +//! `--config` overrides the defaults file by file. + +mod agents; +mod config; +mod profile; +pub mod sdk; +mod skills; + +pub use agents::{AgentDef, GENERAL as GENERAL_AGENT}; +pub use config::{Config, RepoConfig}; +pub use profile::{ + AnthropicCfg, CacheCfg, ContextCfg, EditTool, Effort, GuardsCfg, Masking, OpenAiCfg, Profile, + ShapeCfg, StopCfg, ThinkingBinding, ThinkingDisplay, ToolCalling, ToolNaming, ToolsCfg, +}; +pub use skills::{split_frontmatter, Skill}; + +use std::fmt; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; + +const DEFAULT_CONFIG: &str = include_str!("../../../gitagent.yaml"); +const DEFAULT_SYSTEM_PROMPT: &str = include_str!("../../../prompts/system.md"); +const DEFAULT_LESSONS: &str = include_str!("../../../prompts/lessons.md"); +const DEFAULT_PROFILES: &[(&str, &str)] = &[ + ( + "anthropic", + include_str!("../../../profiles/anthropic.yaml"), + ), + ("openai", include_str!("../../../profiles/openai.yaml")), + ("google", include_str!("../../../profiles/google.yaml")), + ("compat", include_str!("../../../profiles/compat.yaml")), +]; + +/// Model provider family. Profiles are chosen per family. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Family { + Anthropic, + OpenAI, + Google, + Compat, +} + +impl Family { + pub fn as_str(self) -> &'static str { + match self { + Family::Anthropic => "anthropic", + Family::OpenAI => "openai", + Family::Google => "google", + Family::Compat => "compat", + } + } +} + +impl fmt::Display for Family { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A `provider:model` reference, e.g. `openai:`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ModelRef { + pub provider: String, + pub model: String, +} + +impl ModelRef { + pub fn parse(s: &str) -> Result { + match s.split_once(':') { + Some((p, m)) if !p.is_empty() && !m.is_empty() => Ok(ModelRef { + provider: p.to_string(), + model: m.to_string(), + }), + _ => bail!("model must look like provider:model, got {s:?}"), + } + } + + pub fn family(&self) -> Family { + match self.provider.as_str() { + "anthropic" => Family::Anthropic, + "openai" => Family::OpenAI, + "google" | "gemini" => Family::Google, + _ => Family::Compat, + } + } +} + +impl fmt::Display for ModelRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}", self.provider, self.model) + } +} + +/// Everything the agent needs from its configuration files. +#[derive(Debug, Clone)] +pub struct AgentSpec { + pub config: Config, + pub model: ModelRef, + pub profile: Profile, + /// Name or path the profile was loaded from, for the record. + pub profile_source: String, + /// Byte-identical across tasks; nothing task-specific goes here. + pub system_prompt: String, + /// Where `gitagent.yaml` came from (a path, or `built-in`). + pub config_source: String, + /// Directories searched for `prompts/`, `profiles/`, `skills/` and + /// `hooks/`, highest priority first. Empty means built-ins only. + pub config_dirs: Vec, + /// Skills found in the config directories (when `skills: true`). + pub skills: Vec, + /// Sub-agents the `subagent` tool can start (when the profile enables it). + pub agents: Vec, +} + +impl AgentSpec { + /// The first existing `relative` path across the config directories. + pub fn find(&self, relative: &str) -> Option { + find_in(&self.config_dirs, relative) + } +} + +#[derive(Debug, Clone, Default)] +pub struct LoadOptions { + /// Path to a `gitagent.yaml`; its directory is searched first for + /// `prompts/` and `profiles/`. + pub config_path: Option, + /// Look for `gitagent.yaml` (or `.gitagent/gitagent.yaml`) here when no + /// `config_path` is given. Benchmark runs leave this unset. + pub project_dir: Option, + /// The user's home directory; `~/.gitagent/` is the lowest config layer. + /// Benchmark runs leave this unset. + pub home_dir: Option, + /// Overrides `model` from the config. + pub model: Option, + /// Overrides `profile` from the config: `auto`, a profile name or a path to a YAML file. + pub profile: Option, +} + +pub fn load(opts: &LoadOptions) -> Result { + let project_layer = opts.project_dir.as_ref().map(|d| d.join(".gitagent")); + let home_layer = opts.home_dir.as_ref().map(|d| d.join(".gitagent")); + // `.claude/` layers: `.claude/agents`, `.claude/skills` and + // `.claude/settings.json` (hooks) are read the same way, after ours. + let claude_layers: Vec = opts + .project_dir + .iter() + .chain(opts.home_dir.iter()) + .map(|d| d.join(".claude")) + .filter(|d| d.is_dir()) + .collect(); + let config_path = opts.config_path.clone().or_else(|| { + let mut candidates = Vec::new(); + if let Some(d) = &opts.project_dir { + candidates.push(d.join("gitagent.yaml")); + } + candidates.extend(project_layer.iter().map(|d| d.join("gitagent.yaml"))); + candidates.extend(home_layer.iter().map(|d| d.join("gitagent.yaml"))); + candidates.into_iter().find(|p| p.is_file()) + }); + + let mut config_dirs: Vec = Vec::new(); + let layers = config_path + .as_ref() + .map(|p| p.parent().unwrap_or(Path::new(".")).to_path_buf()) + .into_iter() + .chain(project_layer.into_iter().filter(|d| d.is_dir())) + .chain(home_layer.into_iter().filter(|d| d.is_dir())) + .chain(claude_layers); + for dir in layers { + if !config_dirs.contains(&dir) { + config_dirs.push(dir); + } + } + + let (config_text, config_source) = match &config_path { + Some(path) => (read(path)?, path.display().to_string()), + None => (DEFAULT_CONFIG.to_string(), "built-in".to_string()), + }; + let config: Config = serde_saphyr::from_str(&config_text) + .with_context(|| format!("parsing gitagent.yaml ({config_source})"))?; + + let model_str = + opts.model.as_deref().or(config.model.as_deref()).context( + "no model given: pass --model provider:model or set `model` in gitagent.yaml", + )?; + let model = ModelRef::parse(model_str)?; + + let profile_sel = opts.profile.as_deref().unwrap_or(&config.profile); + let (profile, profile_source) = load_profile(profile_sel, model.family(), &config_dirs)?; + + let skills = if config.skills { + skills::discover(&config_dirs) + } else { + Vec::new() + }; + let mut system_prompt = build_system_prompt(&config_dirs)?; + if profile.lessons { + let lessons = lessons(&config_dirs)?; + if !lessons.is_empty() { + system_prompt.push_str("\n# Lessons from earlier tasks\n\n"); + system_prompt.push_str(&lessons); + system_prompt.push('\n'); + } + } + if !skills.is_empty() { + system_prompt.push('\n'); + system_prompt.push_str(&skills::index(&skills)); + } + + let agents = if profile.tools.subagents { + agents::discover(&config_dirs) + } else { + Vec::new() + }; + Ok(AgentSpec { + config, + model, + profile, + profile_source, + system_prompt, + config_source, + agents, + config_dirs, + skills, + }) +} + +/// Only `gitagent.yaml` (explicit path, else `~/.gitagent/gitagent.yaml`, +/// else built-in), for settings needed before a working directory exists. +pub fn load_config(config_path: Option<&Path>, home_dir: Option<&Path>) -> Result { + let path = config_path.map(Path::to_path_buf).or_else(|| { + home_dir + .map(|h| h.join(".gitagent/gitagent.yaml")) + .filter(|p| p.is_file()) + }); + let text = match &path { + Some(p) => read(p)?, + None => DEFAULT_CONFIG.to_string(), + }; + serde_saphyr::from_str(&text).context("parsing gitagent.yaml") +} + +/// The built-in lessons, then each layer's `prompts/lessons.md` (lessons add +/// up rather than override; a file identical to one already taken is skipped). +fn lessons(dirs: &[PathBuf]) -> Result { + let mut parts = vec![strip_header_comment(DEFAULT_LESSONS).trim().to_string()]; + for dir in dirs { + let path = dir.join("prompts/lessons.md"); + if path.is_file() { + let text = strip_header_comment(&read(&path)?).trim().to_string(); + if !parts.contains(&text) { + parts.push(text); + } + } + } + parts.retain(|p| !p.is_empty()); + Ok(parts.join("\n")) +} + +fn find_in(dirs: &[PathBuf], relative: &str) -> Option { + dirs.iter().map(|d| d.join(relative)).find(|p| p.exists()) +} + +fn load_profile(sel: &str, family: Family, dirs: &[PathBuf]) -> Result<(Profile, String)> { + let name = if sel == "auto" { family.as_str() } else { sel }; + let (text, source) = if name.ends_with(".yaml") || name.ends_with(".yml") || name.contains('/') + { + (read(Path::new(name))?, name.to_string()) + } else if let Some(path) = find_in(dirs, &format!("profiles/{name}.yaml")) { + (read(&path)?, path.display().to_string()) + } else if let Some((_, text)) = DEFAULT_PROFILES.iter().find(|(n, _)| *n == name) { + (text.to_string(), format!("built-in:{name}")) + } else { + bail!("unknown profile {name:?}"); + }; + let profile: Profile = + serde_saphyr::from_str(&text).with_context(|| format!("parsing profile {source}"))?; + validate_profile(&profile).with_context(|| format!("profile {source}"))?; + Ok((profile, source)) +} + +/// Combinations that would fail at the provider. +fn validate_profile(p: &Profile) -> Result<()> { + if p.context.masking == Masking::Server && p.family != Family::Anthropic { + bail!("context.masking: server needs an Anthropic model; use client or none"); + } + if p.context.masking == Masking::Client + && p.anthropic + .as_ref() + .is_some_and(|a| a.thinking_binding == ThinkingBinding::Error) + { + bail!( + "context.masking: client edits earlier turns, which Claude rejects while \ + anthropic.thinking_binding is error; set it to drop_block, or use masking: server" + ); + } + if !matches!(p.cache.ttl.as_str(), "5m" | "1h") { + bail!("cache.ttl must be 5m or 1h, got {:?}", p.cache.ttl); + } + Ok(()) +} + +/// `prompts/system.md`, followed by `prompts/rules.md` when present. +fn build_system_prompt(dirs: &[PathBuf]) -> Result { + let system = match find_in(dirs, "prompts/system.md") { + Some(p) => read(&p)?, + None => DEFAULT_SYSTEM_PROMPT.to_string(), + }; + let mut out = strip_header_comment(&system).trim_end().to_string(); + if let Some(rules) = find_in(dirs, "prompts/rules.md") { + out.push_str("\n\n"); + out.push_str(strip_header_comment(&read(&rules)?).trim_end()); + } + out.push('\n'); + Ok(out) +} + +/// Drops a leading `` block (notes for maintainers), so +/// it stays in the file but is never sent to the model. +fn strip_header_comment(text: &str) -> &str { + let trimmed = text.trim_start(); + match trimmed + .strip_prefix("").map(|end| &rest[end + 3..])) + { + Some(rest) => rest.trim_start_matches(['\n', '\r']), + None => text, + } +} + +fn read(path: &Path) -> Result { + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts(model: &str) -> LoadOptions { + LoadOptions { + model: Some(model.into()), + ..Default::default() + } + } + + #[test] + fn built_in_defaults_load_for_every_family() { + for (model, family) in [ + ("anthropic:m", Family::Anthropic), + ("openai:m", Family::OpenAI), + ("google:m", Family::Google), + ("openrouter:org/model", Family::Compat), + ] { + let spec = load(&opts(model)).unwrap(); + assert_eq!(spec.profile.family, family); + assert_eq!(spec.profile_source, format!("built-in:{family}")); + assert_eq!(spec.config.max_turns, 200); + } + } + + #[test] + fn anthropic_profile_sends_no_temperature_and_has_its_options() { + let spec = load(&opts("anthropic:m")).unwrap(); + assert_eq!(spec.profile.temperature, None); + let a = spec.profile.anthropic.expect("anthropic section"); + assert_eq!(a.thinking_display, ThinkingDisplay::Summarized); + assert_eq!(a.thinking_binding, ThinkingBinding::Error); + assert!(a.refusal_fallback); + assert!(load(&opts("openai:m")).unwrap().profile.anthropic.is_none()); + } + + #[test] + fn openai_profile_has_its_options_and_no_temperature() { + let p = load(&opts("openai:m")).unwrap().profile; + assert_eq!(p.temperature, None); + let o = p.openai.expect("openai section"); + assert_eq!(o.reasoning_summary, "auto"); + assert_eq!(o.prompt_cache_key.as_deref(), Some("gitagent-v2")); + } + + #[test] + fn rejects_combinations_the_provider_would_fail() { + let anthropic = DEFAULT_PROFILES[0].1; + let check = |text: String| { + let p: Profile = serde_saphyr::from_str(&text).unwrap(); + validate_profile(&p) + }; + assert!(check(anthropic.to_string()).is_ok()); + let client = anthropic.replace("masking: none", "masking: client"); + let err = check(client.clone()).unwrap_err().to_string(); + assert!(err.contains("drop_block"), "{err}"); + assert!( + check(client.replace("thinking_binding: error", "thinking_binding: drop_block")) + .is_ok() + ); + assert!(check(anthropic.replace("masking: none", "masking: server")).is_ok()); + let openai = DEFAULT_PROFILES[1] + .1 + .replace("masking: none", "masking: server"); + assert!(check(openai).unwrap_err().to_string().contains("Anthropic")); + assert!(check(anthropic.replace("ttl: 5m", "ttl: 10m")).is_err()); + } + + #[test] + fn model_ref_splits_on_first_colon() { + let m = ModelRef::parse("compat:host:port/model").unwrap(); + assert_eq!(m.provider, "compat"); + assert_eq!(m.model, "host:port/model"); + assert!(ModelRef::parse("no-provider").is_err()); + assert!(ModelRef::parse(":x").is_err()); + } + + #[test] + fn missing_model_is_an_error() { + let err = load(&LoadOptions::default()).unwrap_err(); + assert!(err.to_string().contains("no model given")); + } + + /// The prompt without the learned-lessons section, which the committed + /// built-in lessons file adds to every prompt. + fn without_lessons(prompt: &str) -> &str { + prompt + .split("\n# Lessons from earlier tasks\n") + .next() + .unwrap() + } + + #[test] + fn config_dir_overrides_files() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + std::fs::create_dir_all(root.join("prompts")).unwrap(); + std::fs::create_dir_all(root.join("profiles")).unwrap(); + std::fs::write( + root.join("gitagent.yaml"), + "model: anthropic:x\nmax_turns: 7\nprofile: custom\n", + ) + .unwrap(); + std::fs::write(root.join("prompts/system.md"), "SYSTEM\n\n").unwrap(); + std::fs::write(root.join("prompts/rules.md"), "RULES\n").unwrap(); + let custom = DEFAULT_PROFILES[0].1.replace("effort: high", "effort: low"); + std::fs::write(root.join("profiles/custom.yaml"), custom).unwrap(); + + let spec = load(&LoadOptions { + config_path: Some(root.join("gitagent.yaml")), + ..Default::default() + }) + .unwrap(); + assert_eq!(spec.config.max_turns, 7); + assert_eq!(spec.model.to_string(), "anthropic:x"); + assert_eq!(spec.profile.effort, Effort::Low); + assert_eq!(without_lessons(&spec.system_prompt), "SYSTEM\n\nRULES\n"); + } + + #[test] + fn discovers_project_and_home_config_layers() { + let project = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + let home_cfg = home.path().join(".gitagent"); + std::fs::create_dir_all(home_cfg.join("prompts")).unwrap(); + std::fs::write( + home_cfg.join("gitagent.yaml"), + "model: anthropic:home\nmax_turns: 5\n", + ) + .unwrap(); + std::fs::write(home_cfg.join("prompts/rules.md"), "HOME RULES\n").unwrap(); + let opts = LoadOptions { + project_dir: Some(project.path().to_path_buf()), + home_dir: Some(home.path().to_path_buf()), + ..Default::default() + }; + // Only the home config exists. + let spec = load(&opts).unwrap(); + assert_eq!(spec.model.model, "home"); + assert_eq!(spec.config.max_turns, 5); + assert!(without_lessons(&spec.system_prompt).ends_with("HOME RULES\n")); + + // A project config wins, and still picks up home prompts it lacks. + std::fs::write(project.path().join("gitagent.yaml"), "model: openai:proj\n").unwrap(); + let spec = load(&opts).unwrap(); + assert_eq!(spec.model.model, "proj"); + assert_eq!(spec.config.max_turns, 200); + assert!(without_lessons(&spec.system_prompt).ends_with("HOME RULES\n")); + assert_eq!( + spec.config_dirs, + vec![project.path().to_path_buf(), home_cfg.clone()] + ); + assert_eq!( + spec.find("prompts/rules.md"), + Some(home_cfg.join("prompts/rules.md")) + ); + + // Without project or home directories (benchmark runs), only built-ins. + let spec = load(&LoadOptions { + model: Some("anthropic:x".into()), + ..Default::default() + }) + .unwrap(); + assert!(spec.config_dirs.is_empty()); + assert_eq!(spec.config_source, "built-in"); + } + + #[test] + fn unknown_config_keys_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("gitagent.yaml"); + std::fs::write(&path, "model: anthropic:x\nmax_turn: 7\n").unwrap(); + let err = load(&LoadOptions { + config_path: Some(path), + ..Default::default() + }) + .unwrap_err(); + assert!(format!("{err:#}").contains("max_turn"), "{err:#}"); + } + + #[test] + fn lessons_are_appended_when_present_and_enabled() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("prompts")).unwrap(); + std::fs::write(dir.path().join("gitagent.yaml"), "model: anthropic:x\n").unwrap(); + std::fs::write( + dir.path().join("prompts/lessons.md"), + "\n- Start servers so they outlive the command.\n", + ) + .unwrap(); + let opts = LoadOptions { + config_path: Some(dir.path().join("gitagent.yaml")), + ..Default::default() + }; + let spec = load(&opts).unwrap(); + // The project's lessons follow the built-in ones under one heading. + assert!(spec.system_prompt.contains( + " +# Lessons from earlier tasks + +- " + )); + assert!(spec.system_prompt.ends_with( + "- Start servers so they outlive the command. +" + )); + assert_eq!( + spec.system_prompt + .matches("# Lessons from earlier tasks") + .count(), + 1 + ); + // With lessons off, none of it is added. + let dir2 = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir2.path().join("profiles")).unwrap(); + std::fs::write( + dir2.path().join("gitagent.yaml"), + "model: anthropic:x +profile: quiet +", + ) + .unwrap(); + std::fs::write( + dir2.path().join("profiles/quiet.yaml"), + include_str!("../../../profiles/anthropic.yaml") + .replace("lessons: true", "lessons: false"), + ) + .unwrap(); + let plain = load(&LoadOptions { + config_path: Some(dir2.path().join("gitagent.yaml")), + ..Default::default() + }) + .unwrap(); + assert!(!plain.system_prompt.contains("Lessons from earlier tasks")); + } + + #[test] + fn claude_code_layers_are_read_after_ours() { + let project = tempfile::tempdir().unwrap(); + let claude = project.path().join(".claude"); + std::fs::create_dir_all(claude.join("agents")).unwrap(); + std::fs::create_dir_all(claude.join("skills/pdf")).unwrap(); + std::fs::write( + claude.join("agents/reviewer.md"), + "---\nname: reviewer\ndescription: Reviews.\ntools: Read, Bash\nmaxTurns: 5\n---\nReview.\n", + ) + .unwrap(); + std::fs::write( + claude.join("skills/pdf/SKILL.md"), + "---\nname: pdf\ndescription: PDF forms.\n---\nUse pypdf.\n", + ) + .unwrap(); + // Our own layer still comes first for the same name. + std::fs::create_dir_all(project.path().join(".gitagent/skills/pdf")).unwrap(); + std::fs::write( + project.path().join(".gitagent/skills/pdf/SKILL.md"), + "---\nname: pdf\ndescription: Ours wins.\n---\nx\n", + ) + .unwrap(); + std::fs::write( + project.path().join(".gitagent/gitagent.yaml"), + "model: anthropic:x\nprofile: anthropic\n", + ) + .unwrap(); + let mut spec = load(&LoadOptions { + project_dir: Some(project.path().to_path_buf()), + ..Default::default() + }) + .unwrap(); + assert_eq!(spec.config_dirs.last().unwrap(), &claude); + assert_eq!(spec.skills.len(), 1); + assert_eq!(spec.skills[0].description, "Ours wins."); + // Sub-agent files are read when the profile enables sub-agents. + spec.profile.tools.subagents = true; + let agents = agents::discover(&spec.config_dirs); + let r = agents.iter().find(|a| a.name == "reviewer").unwrap(); + assert_eq!(r.max_turns, 5); + assert_eq!( + r.tools.as_deref(), + Some( + &[ + "read_file".to_string(), + "image_read".to_string(), + "exec".to_string() + ][..] + ) + ); + // Benchmark runs have no project or home, so none of this applies. + let bench = load(&LoadOptions { + model: Some("anthropic:x".into()), + ..Default::default() + }) + .unwrap(); + assert!(bench.config_dirs.is_empty()); + } + + #[test] + fn lessons_from_every_layer_add_up() { + let project = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + for (dir, lesson) in [ + (project.path(), "- From the project.\n"), + (home.path(), "- From home.\n"), + ] { + let prompts = dir.join(".gitagent/prompts"); + std::fs::create_dir_all(&prompts).unwrap(); + std::fs::write(prompts.join("lessons.md"), format!("\n{lesson}")).unwrap(); + } + let spec = load(&LoadOptions { + model: Some("anthropic:x".into()), + project_dir: Some(project.path().to_path_buf()), + home_dir: Some(home.path().to_path_buf()), + ..Default::default() + }) + .unwrap(); + assert!(spec.system_prompt.ends_with( + "- From the project. +- From home. +" + )); + } + + #[test] + fn header_comments_are_not_sent_to_the_model() { + let prompt = load(&opts("anthropic:a")).unwrap().system_prompt; + assert!(prompt.starts_with("# Role and environment"), "{prompt}"); + assert!(!prompt.contains("\n\nBody\n"), "Body\n"); + assert_eq!( + strip_header_comment("Body "), + "Body " + ); + } + + #[test] + fn system_prompt_does_not_depend_on_the_model() { + let a = load(&opts("anthropic:a")).unwrap().system_prompt; + let b = load(&opts("openai:b")).unwrap().system_prompt; + assert_eq!(a, b); + } +} diff --git a/crates/loader/src/profile.rs b/crates/loader/src/profile.rs new file mode 100644 index 00000000..3306196d --- /dev/null +++ b/crates/loader/src/profile.rs @@ -0,0 +1,235 @@ +use serde::Deserialize; + +use crate::Family; + +/// Per-model-family tuning (BLUEPRINT §6). Every field is required so that a +/// profile always states its values explicitly. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Profile { + pub family: Family, + pub tool_calling: ToolCalling, + /// Which names the model sees for the tools: gitagent's own, or the + /// `.claude/` ones (`Bash`, `Read`, `Edit`, … with their argument names). + #[serde(default)] + pub tool_naming: ToolNaming, + pub edit_tool: EditTool, + pub effort: Effort, + /// Omitted for models that reject a temperature. + #[serde(default)] + pub temperature: Option, + pub max_output_tokens: u32, + pub context_window: u32, + pub exec_default_yield_ms: u64, + pub shape: ShapeCfg, + pub context: ContextCfg, + pub guards: GuardsCfg, + /// Probe the environment once and add it to the first user message. + pub snapshot: bool, + /// Append `prompts/lessons.md` (learned lessons) to the system prompt. + pub lessons: bool, + pub stop: StopCfg, + pub cache: CacheCfg, + pub tools: ToolsCfg, + /// Anthropic-only request options; required when `family: anthropic`. + #[serde(default)] + pub anthropic: Option, + /// OpenAI-only request options; required when `family: openai`. + #[serde(default)] + pub openai: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolCalling { + Native, + NativeWithTextFallback, + Text, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolNaming { + #[default] + Gitagent, + Sdk, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EditTool { + StrReplace, + ApplyPatch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Effort { + Low, + Medium, + High, + #[serde(rename = "xhigh")] + XHigh, + Max, +} + +impl Effort { + /// One level higher (`max` stays `max`). + pub fn next(self) -> Effort { + match self { + Effort::Low => Effort::Medium, + Effort::Medium => Effort::High, + Effort::High => Effort::XHigh, + Effort::XHigh | Effort::Max => Effort::Max, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Effort::Low => "low", + Effort::Medium => "medium", + Effort::High => "high", + Effort::XHigh => "xhigh", + Effort::Max => "max", + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ShapeCfg { + pub cap_tokens: u32, + pub head_ratio: f32, + pub collapse_min_run: u32, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContextCfg { + pub masking: Masking, + pub mask_batch_tokens: u32, + pub keep_last: u32, + /// Summarize the history when fewer than `compact_at_tokens` of the + /// context window are left (and when the provider reports overflow). + pub compaction: bool, + pub compact_at_tokens: u32, +} + +/// How old tool results are hidden. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Masking { + None, + /// gitagent replaces them with stubs at batched checkpoints. + Client, + /// The provider clears them on its side (Anthropic context editing); the + /// history sent stays unchanged. + Server, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GuardsCfg { + /// Minutes left in every tool result, a reminder at `nudge_at` and a + /// final call at `force_stop_at` (shares of the time budget). + pub time_budget: bool, + pub nudge_at: f32, + pub force_stop_at: f32, + pub loop_detector: bool, + pub loop_repeats: u32, + /// After a loop warning, use the next effort level for one call. + pub loop_effort_bump: bool, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StopCfg { + pub enabled: bool, + pub max_rejections: u32, + pub verify_patterns: Vec, + pub edit_patterns: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CacheCfg { + pub breakpoints: Vec, + /// Cache lifetime where the provider lets us choose: `5m` or `1h`. + #[serde(default = "default_ttl")] + pub ttl: String, +} + +fn default_ttl() -> String { + "5m".into() +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AnthropicCfg { + /// `summarized` returns a readable summary of the model's reasoning (kept in + /// the trajectory); `omitted` returns empty thinking text. Billing is the same. + pub thinking_display: ThinkingDisplay, + /// What the API does with a replayed thinking block whose conversation + /// prefix changed: `error` (reject the request) or `drop_block`. + pub thinking_binding: ThinkingBinding, + /// Let the API re-run a refused request on its recommended fallback model. + pub refusal_fallback: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ThinkingDisplay { + Summarized, + Omitted, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ThinkingBinding { + Error, + DropBlock, +} + +/// Which optional tools the model is offered. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ToolsCfg { + /// Background sessions (`exec` with `background`) and the `write_stdin` + /// tool for polling, input and Ctrl-C. + pub write_stdin: bool, + /// `image_read`: show the model an image file (vision models only). + pub image_read: bool, + /// `todo`: a checklist the model keeps for itself. + pub todo: bool, + /// `subagent`: start a sub-agent with a fresh context for a + /// self-contained piece of work; only its summary comes back. + #[serde(default)] + pub subagents: bool, + /// How many levels of sub-agents may nest (1: sub-agents cannot start + /// sub-agents of their own). + #[serde(default = "default_subagent_depth")] + pub subagent_depth: u32, + /// Sub-agents called in one reply run at the same time, this many at once. + #[serde(default = "default_max_concurrent_subagents")] + pub max_concurrent_subagents: u32, +} + +fn default_subagent_depth() -> u32 { + 1 +} + +fn default_max_concurrent_subagents() -> u32 { + 4 +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OpenAiCfg { + /// Reasoning summary to request (`auto`, `concise`, `detailed`), or + /// `none` to request none. Summaries make trajectories readable. + pub reasoning_summary: String, + /// Stable cache-routing key shared by every task, so requests with the + /// same system prompt and tools land on the same prompt cache. + #[serde(default)] + pub prompt_cache_key: Option, +} diff --git a/crates/loader/src/sdk.rs b/crates/loader/src/sdk.rs new file mode 100644 index 00000000..770de487 --- /dev/null +++ b/crates/loader/src/sdk.rs @@ -0,0 +1,256 @@ +//! The tool and argument names used in `.claude/` agent files, hook settings +//! and hook scripts, so that those files work with gitagent unchanged. +//! +//! gitagent's own names (`exec`, `read_file`, …) stay what the model sees; +//! these tables translate at the edges: when reading `.claude/agents/*.md`, +//! when matching and calling hooks from `.claude/settings.json`, and when +//! applying a hook's `updatedInput`. + +use serde_json::{Map, Value}; + +/// gitagent tools that a `.claude/` tool name stands for. Names with no +/// equivalent (Glob, Grep, WebFetch, …) give an empty list. +pub fn gitagent_tools_for(sdk_name: &str) -> &'static [&'static str] { + match sdk_name { + "Bash" => &["exec"], + "Read" => &["read_file", "image_read"], + "Write" => &["write_file"], + "Edit" | "MultiEdit" => &["str_replace", "apply_patch"], + "Agent" | "Task" => &["subagent"], + "Skill" => &["skill"], + "TodoWrite" => &["todo"], + _ => &[], + } +} + +/// The `.claude/` name for a gitagent tool; tools without one keep their own. +pub fn sdk_name(tool: &str) -> &str { + match tool { + "exec" => "Bash", + "read_file" | "image_read" => "Read", + "write_file" => "Write", + "str_replace" | "apply_patch" => "Edit", + "subagent" => "Agent", + "skill" => "Skill", + "todo" => "TodoWrite", + other => other, + } +} + +/// The model id an alias stands for (`sonnet`, `opus`, `haiku`, +/// `fable`); anything else is already an id. +pub fn model_alias(name: &str) -> &str { + match name { + "sonnet" => "claude-sonnet-5-5", + "opus" => "claude-opus-5-5", + "haiku" => "claude-haiku-5-5", + "fable" => "claude-fable-5-1", + other => other, + } +} + +/// A tool list from a `.claude/` agent file, as gitagent tool names: +/// `.claude/` names are expanded, gitagent names pass through, names with no +/// equivalent are dropped. +pub fn normalize_tool_list(names: &[String]) -> Vec { + let mut out: Vec = Vec::new(); + for name in names { + let name = name.trim(); + if name.is_empty() { + continue; + } + let mapped = gitagent_tools_for(name); + let expanded: Vec = if mapped.is_empty() { + if is_gitagent_tool(name) { + vec![name.to_string()] + } else { + Vec::new() + } + } else { + mapped.iter().map(|s| s.to_string()).collect() + }; + for t in expanded { + if !out.contains(&t) { + out.push(t); + } + } + } + out +} + +fn is_gitagent_tool(name: &str) -> bool { + matches!( + name, + "exec" + | "write_stdin" + | "read_file" + | "write_file" + | "str_replace" + | "apply_patch" + | "image_read" + | "todo" + | "task_complete" + | "skill" + | "subagent" + ) +} + +/// The gitagent tool behind a call the model made under `.claude/` names, with +/// its arguments translated. `Read` of an image file is `image_read`; +/// `Edit` is the profile's edit tool. Names that are already gitagent's pass +/// through; unknown names come back unchanged. +pub fn gitagent_tool_for_call( + name: &str, + args: &Map, +) -> (String, Map) { + let target: &str = match name { + "Bash" => "exec", + "Read" => { + let path = args.get("file_path").and_then(Value::as_str).unwrap_or(""); + let ext = path.rsplit('.').next().unwrap_or("").to_ascii_lowercase(); + if matches!(ext.as_str(), "png" | "jpg" | "jpeg" | "gif" | "webp") { + "image_read" + } else { + "read_file" + } + } + "Write" => "write_file", + "Edit" | "MultiEdit" => "str_replace", + "Agent" | "Task" => "subagent", + "Skill" => "skill", + "TodoWrite" => "todo", + other => return (other.to_string(), args.clone()), + }; + (target.to_string(), from_sdk_input(target, args)) +} + +/// (gitagent argument, `.claude/` argument) pairs for a gitagent tool. +pub fn renames(tool: &str) -> &'static [(&'static str, &'static str)] { + match tool { + "exec" => &[ + ("cmd", "command"), + ("yield_ms", "timeout"), + ("background", "run_in_background"), + ], + "read_file" | "image_read" | "write_file" => &[("path", "file_path")], + "str_replace" => &[ + ("path", "file_path"), + ("old_str", "old_string"), + ("new_str", "new_string"), + ], + "subagent" => &[("agent", "subagent_type")], + "skill" => &[("name", "skill")], + _ => &[], + } +} + +/// A tool call's arguments with the `.claude/` argument names. +pub fn to_sdk_input(tool: &str, args: &Map) -> Map { + rename(args, renames(tool).iter().map(|(ours, sdk)| (*ours, *sdk))) +} + +/// Arguments written with the `.claude/` names, as gitagent's. +pub fn from_sdk_input(tool: &str, args: &Map) -> Map { + rename(args, renames(tool).iter().map(|(ours, sdk)| (*sdk, *ours))) +} + +fn rename<'a>( + args: &Map, + pairs: impl Iterator, +) -> Map { + let pairs: Vec<(&str, &str)> = pairs.collect(); + let mut out = Map::new(); + for (k, v) in args { + let key = pairs + .iter() + .find(|(from, _)| *from == k) + .map(|(_, to)| to.to_string()) + .unwrap_or_else(|| k.clone()); + out.insert(key, v.clone()); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn tool_lists_translate() { + let list = |s: &str| { + s.split(',') + .map(|x| x.trim().to_string()) + .collect::>() + }; + assert_eq!( + normalize_tool_list(&list("Read, Grep, Bash, exec, Nope")), + vec!["read_file", "image_read", "exec"] + ); + assert_eq!( + normalize_tool_list(&list("Edit")), + vec!["str_replace", "apply_patch"] + ); + assert_eq!(sdk_name("exec"), "Bash"); + assert_eq!(sdk_name("task_complete"), "task_complete"); + } + + #[test] + fn calls_under_sdk_names_map_back() { + let args = |v: Value| v.as_object().unwrap().clone(); + let (t, a) = + gitagent_tool_for_call("Bash", &args(json!({"command": "ls", "timeout": 100}))); + assert_eq!( + (t.as_str(), Value::Object(a)), + ("exec", json!({"cmd": "ls", "yield_ms": 100})) + ); + let (t, a) = gitagent_tool_for_call("Read", &args(json!({"file_path": "/tmp/shot.PNG"}))); + assert_eq!( + (t.as_str(), Value::Object(a)), + ("image_read", json!({"path": "/tmp/shot.PNG"})) + ); + let (t, _) = + gitagent_tool_for_call("Read", &args(json!({"file_path": "notes.md", "offset": 3}))); + assert_eq!(t, "read_file"); + let (t, a) = gitagent_tool_for_call( + "Edit", + &args(json!({"file_path": "a", "old_string": "x", "new_string": "y"})), + ); + assert_eq!( + (t.as_str(), Value::Object(a)), + ( + "str_replace", + json!({"path": "a", "old_str": "x", "new_str": "y"}) + ) + ); + let (t, a) = gitagent_tool_for_call("task_complete", &args(json!({"summary": "s"}))); + assert_eq!( + (t.as_str(), Value::Object(a)), + ("task_complete", json!({"summary": "s"})) + ); + let (t, _) = gitagent_tool_for_call("Glob", &Map::new()); + assert_eq!(t, "Glob"); + } + + #[test] + fn arguments_translate_both_ways() { + let ours = json!({"cmd": "ls", "yield_ms": 5000}) + .as_object() + .unwrap() + .clone(); + let sdk = to_sdk_input("exec", &ours); + assert_eq!( + Value::Object(sdk.clone()), + json!({"command": "ls", "timeout": 5000}) + ); + assert_eq!(from_sdk_input("exec", &sdk), ours); + let edit = json!({"file_path": "a.py", "old_string": "x", "new_string": "y"}); + assert_eq!( + Value::Object(from_sdk_input("str_replace", edit.as_object().unwrap())), + json!({"path": "a.py", "old_str": "x", "new_str": "y"}) + ); + // Unknown keys pass through untouched. + let other = json!({"summary": "done"}).as_object().unwrap().clone(); + assert_eq!(to_sdk_input("task_complete", &other), other); + } +} diff --git a/crates/loader/src/skills.rs b/crates/loader/src/skills.rs new file mode 100644 index 00000000..7a1456b9 --- /dev/null +++ b/crates/loader/src/skills.rs @@ -0,0 +1,130 @@ +//! Skills: `skills//SKILL.md` files in a config directory. Their names +//! and descriptions go in the system prompt; the `skill` tool loads the full +//! text when the agent needs it. + +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Skill { + pub name: String, + pub description: String, + pub path: PathBuf, +} + +/// Skills from all config directories; for the same name the first directory +/// wins. Sorted by name, so the prompt index is stable. +pub fn discover(config_dirs: &[PathBuf]) -> Vec { + let mut skills: Vec = Vec::new(); + for dir in config_dirs { + let Ok(entries) = std::fs::read_dir(dir.join("skills")) else { + continue; + }; + let mut found: Vec = entries + .filter_map(|e| e.ok()) + .map(|e| e.path().join("SKILL.md")) + .filter(|p| p.is_file()) + .collect(); + found.sort(); + for path in found { + if let Some(skill) = read(&path) { + if !skills.iter().any(|s| s.name == skill.name) { + skills.push(skill); + } + } + } + } + skills.sort_by(|a, b| a.name.cmp(&b.name)); + skills +} + +fn read(path: &Path) -> Option { + let text = std::fs::read_to_string(path).ok()?; + let folder = path.parent()?.file_name()?.to_string_lossy().to_string(); + let (front, body) = split_frontmatter(&text); + let field = |key: &str| { + front.lines().find_map(|l| { + l.strip_prefix(key) + .and_then(|rest| rest.trim_start().strip_prefix(':')) + .map(|v| v.trim().trim_matches('"').to_string()) + }) + }; + let name = field("name").filter(|n| !n.is_empty()).unwrap_or(folder); + let description = field("description") + .filter(|d| !d.is_empty()) + .or_else(|| { + body.lines() + .map(|l| l.trim().trim_start_matches('#').trim()) + .find(|l| !l.is_empty()) + .map(str::to_string) + }) + .unwrap_or_default(); + Some(Skill { + name, + description: description.chars().take(300).collect(), + path: path.to_path_buf(), + }) +} + +/// Splits a leading `---` ... `---` block from the body. +pub fn split_frontmatter(text: &str) -> (&str, &str) { + let Some(rest) = text.strip_prefix("---") else { + return ("", text); + }; + match rest.find("\n---") { + Some(end) => { + let body = &rest[end + 4..]; + (&rest[..end], body.strip_prefix('\n').unwrap_or(body)) + } + None => ("", text), + } +} + +/// The section added to the system prompt. +pub fn index(skills: &[Skill]) -> String { + let mut out = String::from( + "# Skills\n\nLoad a skill with the `skill` tool when it fits the task; it gives detailed instructions.\n", + ); + for s in skills { + out.push_str(&format!("- {}: {}\n", s.name, s.description)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovers_skills_with_and_without_frontmatter() { + let a = tempfile::tempdir().unwrap(); + let b = tempfile::tempdir().unwrap(); + let write = |root: &Path, name: &str, text: &str| { + std::fs::create_dir_all(root.join("skills").join(name)).unwrap(); + std::fs::write(root.join("skills").join(name).join("SKILL.md"), text).unwrap(); + }; + write( + a.path(), + "pdf", + "---\nname: pdf\ndescription: Read and fill PDF forms.\n---\n# PDF\nUse pypdf.\n", + ); + write(a.path(), "zeta", "# Zeta tricks\nBody.\n"); + write( + b.path(), + "pdf", + "---\nname: pdf\ndescription: shadowed\n---\n", + ); + write( + b.path(), + "alpha", + "---\ndescription: \"Alpha things\"\n---\nx\n", + ); + let skills = discover(&[a.path().to_path_buf(), b.path().to_path_buf()]); + let names: Vec<&str> = skills.iter().map(|s| s.name.as_str()).collect(); + assert_eq!(names, ["alpha", "pdf", "zeta"]); + assert_eq!(skills[0].description, "Alpha things"); + assert_eq!(skills[1].description, "Read and fill PDF forms."); + assert_eq!(skills[2].description, "Zeta tricks"); + assert!(index(&skills).contains("- pdf: Read and fill PDF forms.\n")); + assert_eq!(split_frontmatter("---\na: 1\n---\nbody").1, "body"); + } +} diff --git a/crates/provider/Cargo.toml b/crates/provider/Cargo.toml new file mode 100644 index 00000000..b975c2a5 --- /dev/null +++ b/crates/provider/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "gitagent-provider" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +async-trait.workspace = true +gitagent-loader.workspace = true +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls", "http2"] } +serde.workspace = true +serde_json.workspace = true +serde-saphyr = { version = "1.3.0", default-features = false, features = ["deserialize"] } +thiserror.workspace = true + +[dev-dependencies] +tokio.workspace = true diff --git a/crates/provider/src/anthropic.rs b/crates/provider/src/anthropic.rs new file mode 100644 index 00000000..517ae146 --- /dev/null +++ b/crates/provider/src/anthropic.rs @@ -0,0 +1,462 @@ +//! Anthropic Messages API adapter: raw HTTP with streaming responses. +//! +//! Assistant turns keep the API's own content blocks (`Message::raw`) and are +//! replayed verbatim, because thinking blocks are bound to their signature +//! and to the exact conversation that produced them. The history is append-only. + +use std::collections::BTreeMap; +use std::time::Duration; + +use async_trait::async_trait; +use gitagent_loader::{AnthropicCfg, Family, ThinkingBinding, ThinkingDisplay}; +use serde_json::{json, Map, Value}; + +use crate::sse::SseDecoder; +use crate::{ + ChatRequest, ChatResponse, Content, Message, Provider, ProviderError, RawContent, Role, + StopReason, ToolCall, Usage, +}; + +pub const PROVIDER: &str = "anthropic"; +const DEFAULT_BASE_URL: &str = "https://api.anthropic.com"; +const API_VERSION: &str = "2023-06-01"; +const BETA_THINKING_BINDING: &str = "thinking-binding-controls-2026-08-01"; +const BETA_FALLBACK: &str = "server-side-fallback-2026-07-01"; +const BETA_CONTEXT_EDITING: &str = "context-management-2025-06-27"; +/// Block types that may carry a `cache_control` marker. +const CACHEABLE: &[&str] = &["text", "tool_use", "tool_result", "image", "document"]; + +pub struct AnthropicProvider { + client: reqwest::Client, + base_url: String, + api_key: String, + cfg: AnthropicCfg, +} + +impl AnthropicProvider { + pub fn new( + api_key: String, + base_url: Option, + cfg: AnthropicCfg, + ) -> anyhow::Result { + let client = reqwest::Client::builder() + .user_agent(concat!("gitagent/", env!("CARGO_PKG_VERSION"))) + .connect_timeout(Duration::from_secs(30)) + // The stream sends pings, so a long silence means a dead connection. + .read_timeout(Duration::from_secs(300)) + .build()?; + Ok(AnthropicProvider { + client, + base_url: base_url + .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) + .trim_end_matches('/') + .to_string(), + api_key, + cfg, + }) + } + + /// Reads `ANTHROPIC_API_KEY` and, if set, `ANTHROPIC_BASE_URL`. + pub fn from_env(cfg: AnthropicCfg) -> anyhow::Result { + let key = std::env::var("ANTHROPIC_API_KEY") + .ok() + .filter(|k| !k.trim().is_empty()) + .ok_or_else(|| anyhow::anyhow!("ANTHROPIC_API_KEY is not set"))?; + Self::new(key, std::env::var("ANTHROPIC_BASE_URL").ok(), cfg) + } +} + +#[async_trait] +impl Provider for AnthropicProvider { + async fn complete(&self, req: &ChatRequest) -> Result { + let body = request_body(req, &self.cfg); + let mut http = self + .client + .post(format!("{}/v1/messages", self.base_url)) + .header("x-api-key", &self.api_key) + .header("anthropic-version", API_VERSION) + .header("content-type", "application/json") + .header("accept", "text/event-stream"); + let mut betas = betas(&self.cfg); + if req.server_masking.is_some() { + betas.push(BETA_CONTEXT_EDITING); + } + if !betas.is_empty() { + http = http.header("anthropic-beta", betas.join(",")); + } + let body = serde_json::to_vec(&body).map_err(|e| ProviderError::Fatal(e.to_string()))?; + let mut resp = http + .body(body) + .send() + .await + .map_err(|e| ProviderError::Transient(format!("request failed: {e}")))?; + + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + return Err(classify_http_error(status.as_u16(), &text)); + } + let mut stream = StreamParser::default(); + while let Some(chunk) = resp + .chunk() + .await + .map_err(|e| ProviderError::Transient(format!("stream interrupted: {e}")))? + { + stream.feed(&chunk)?; + } + stream.finish() + } + + fn family(&self) -> Family { + Family::Anthropic + } +} + +fn betas(cfg: &AnthropicCfg) -> Vec<&'static str> { + let mut betas = vec![BETA_THINKING_BINDING]; + if cfg.refusal_fallback { + betas.push(BETA_FALLBACK); + } + betas +} + +/// Builds the request JSON. Serialization is deterministic (sorted keys), so +/// the bytes of the shared prefix are identical from one turn to the next. +pub(crate) fn request_body(req: &ChatRequest, cfg: &AnthropicCfg) -> Value { + let marker = if req.cache.one_hour { + json!({"type": "ephemeral", "ttl": "1h"}) + } else { + json!({"type": "ephemeral"}) + }; + let mut system = json!({"type": "text", "text": req.system}); + if req.cache.system { + system["cache_control"] = marker.clone(); + } + + let tools: Vec = req + .tools + .iter() + .map(|t| json!({"name": t.name, "description": t.description, "input_schema": t.input_schema})) + .collect(); + + let mut messages: Vec = req + .messages + .iter() + .map(|m| { + json!({ + "role": match m.role { Role::User => "user", Role::Assistant => "assistant" }, + "content": message_blocks(m), + }) + }) + .collect(); + let n = messages.len(); + for msg in messages + .iter_mut() + .skip(n.saturating_sub(req.cache.last_messages as usize)) + { + mark_last_cacheable(msg, &marker); + } + + let mut thinking = json!({ + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": match cfg.thinking_binding { + ThinkingBinding::Error => "error", + ThinkingBinding::DropBlock => "drop_block", + } + }, + }); + // `omitted` is the API default, so only `summarized` needs sending. + if cfg.thinking_display == ThinkingDisplay::Summarized { + thinking["display"] = json!("summarized"); + } + + let mut body = json!({ + "model": req.model, + "max_tokens": req.max_output_tokens, + "stream": true, + "system": [system], + "tools": tools, + "messages": messages, + "thinking": thinking, + "output_config": {"effort": req.effort.as_str()}, + }); + if let Some(t) = req.temperature { + body["temperature"] = crate::temperature_json(t); + } + if cfg.refusal_fallback { + body["fallbacks"] = json!("default"); + } + if let Some(m) = req.server_masking { + body["context_management"] = json!({"edits": [{ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": m.trigger_tokens}, + "keep": {"type": "tool_uses", "value": m.keep}, + "clear_at_least": {"type": "input_tokens", "value": m.clear_at_least}, + }]}); + } + body +} + +fn message_blocks(m: &Message) -> Vec { + if let Some(raw) = m.raw.as_ref().filter(|r| r.provider == PROVIDER) { + return raw.blocks.clone(); + } + m.content + .iter() + .map(|c| match c { + Content::Text { text } => json!({"type": "text", "text": text}), + Content::ToolUse { id, name, input } => { + json!({"type": "tool_use", "id": id, "name": name, "input": input}) + } + Content::ToolResult { + tool_use_id, + content, + is_error, + images, + } => { + let content = if images.is_empty() { + json!(content) + } else { + let mut parts = vec![json!({"type": "text", "text": content})]; + parts.extend(images.iter().map(|i| { + json!({"type": "image", "source": {"type": "base64", "media_type": i.media_type, "data": i.data}}) + })); + Value::Array(parts) + }; + let mut block = + json!({"type": "tool_result", "tool_use_id": tool_use_id, "content": content}); + if *is_error { + block["is_error"] = json!(true); + } + block + } + }) + .collect() +} + +/// Puts a cache breakpoint on the last block of `msg` that can carry one. +fn mark_last_cacheable(msg: &mut Value, marker: &Value) { + let Some(blocks) = msg["content"].as_array_mut() else { + return; + }; + if let Some(block) = blocks + .iter_mut() + .rev() + .find(|b| CACHEABLE.contains(&b["type"].as_str().unwrap_or(""))) + { + block["cache_control"] = marker.clone(); + } +} + +pub(crate) fn classify_http_error(status: u16, body: &str) -> ProviderError { + let parsed: Value = serde_json::from_str(body).unwrap_or(Value::Null); + let kind = parsed["error"]["type"].as_str().unwrap_or(""); + let message = parsed["error"]["message"] + .as_str() + .map(str::to_string) + .unwrap_or_else(|| body.chars().take(500).collect()); + let detail = format!("HTTP {status} {kind}: {message}"); + classify(status, kind, &message, detail) +} + +fn classify(status: u16, kind: &str, message: &str, detail: String) -> ProviderError { + let lower = message.to_lowercase(); + if matches!(status, 408 | 429 | 500 | 502 | 503 | 504 | 529) + || matches!(kind, "rate_limit_error" | "api_error" | "overloaded_error") + { + ProviderError::Transient(detail) + } else if kind == "request_too_large" + || lower.contains("prompt is too long") + || lower.contains("context window") + || lower.contains("too many tokens") + { + ProviderError::ContextOverflow(detail) + } else { + ProviderError::Fatal(detail) + } +} + +/// Assembles a streamed response from server-sent events. +#[derive(Default)] +pub(crate) struct StreamParser { + sse: SseDecoder, + model: Option, + blocks: Vec, + partial_json: BTreeMap, + usage: Map, + stop_reason: Option, + cleared_tool_results: u64, + done: bool, +} + +impl StreamParser { + pub fn feed(&mut self, bytes: &[u8]) -> Result<(), ProviderError> { + for data in self.sse.feed(bytes) { + self.handle_event(&data)?; + } + Ok(()) + } + + fn handle_event(&mut self, data: &str) -> Result<(), ProviderError> { + let ev: Value = serde_json::from_str(data) + .map_err(|e| ProviderError::Transient(format!("bad stream event: {e}")))?; + let index = ev["index"].as_u64().unwrap_or(0) as usize; + match ev["type"].as_str().unwrap_or("") { + "message_start" => { + let msg = &ev["message"]; + self.model = msg["model"].as_str().map(str::to_string); + self.merge_usage(&msg["usage"]); + } + "content_block_start" => { + if self.blocks.len() <= index { + self.blocks.resize(index + 1, Value::Null); + } + self.blocks[index] = ev["content_block"].clone(); + } + "content_block_delta" => { + let delta = &ev["delta"]; + let Some(block) = self.blocks.get_mut(index) else { + return Ok(()); + }; + match delta["type"].as_str().unwrap_or("") { + "text_delta" => append(block, "text", &delta["text"]), + "thinking_delta" => append(block, "thinking", &delta["thinking"]), + "signature_delta" => block["signature"] = delta["signature"].clone(), + "input_json_delta" => self + .partial_json + .entry(index) + .or_default() + .push_str(delta["partial_json"].as_str().unwrap_or("")), + _ => {} + } + } + "content_block_stop" => { + if let (Some(json), Some(block)) = + (self.partial_json.remove(&index), self.blocks.get_mut(index)) + { + if !json.trim().is_empty() { + // A call cut off by max_tokens has incomplete JSON; it + // goes back to the model as a call with no arguments. + block["input"] = serde_json::from_str::(&json) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| json!({})); + } + } + } + "message_delta" => { + if let Some(reason) = ev["delta"]["stop_reason"].as_str() { + self.stop_reason = Some(reason.to_string()); + } + for edit in ev["context_management"]["applied_edits"] + .as_array() + .into_iter() + .flatten() + { + self.cleared_tool_results += edit["cleared_tool_uses"].as_u64().unwrap_or(0); + } + self.merge_usage(&ev["usage"]); + } + "message_stop" => self.done = true, + "error" => { + let kind = ev["error"]["type"].as_str().unwrap_or(""); + let message = ev["error"]["message"].as_str().unwrap_or(""); + return Err(classify( + 0, + kind, + message, + format!("stream error {kind}: {message}"), + )); + } + _ => {} + } + Ok(()) + } + + fn merge_usage(&mut self, usage: &Value) { + if let Some(obj) = usage.as_object() { + for (k, v) in obj { + if v.is_number() { + self.usage.insert(k.clone(), v.clone()); + } + } + } + } + + pub fn finish(self) -> Result { + if !self.done { + return Err(ProviderError::Transient( + "stream ended before message_stop".into(), + )); + } + let count = |k: &str| self.usage.get(k).and_then(Value::as_u64).unwrap_or(0); + let cache_read = count("cache_read_input_tokens"); + let cache_write = count("cache_creation_input_tokens"); + let usage = Usage { + input_tokens: count("input_tokens") + cache_read + cache_write, + output_tokens: count("output_tokens"), + cache_read_tokens: cache_read, + cache_write_tokens: cache_write, + }; + + let mut blocks: Vec = self.blocks.into_iter().filter(|b| !b.is_null()).collect(); + let mut text = Vec::new(); + let mut reasoning = Vec::new(); + let mut tool_calls = Vec::new(); + for b in &blocks { + match b["type"].as_str().unwrap_or("") { + "text" => text.push(b["text"].as_str().unwrap_or("").to_string()), + "thinking" => { + let t = b["thinking"].as_str().unwrap_or(""); + if !t.is_empty() { + reasoning.push(t.to_string()); + } + } + "tool_use" => tool_calls.push(ToolCall { + id: b["id"].as_str().unwrap_or("").to_string(), + name: b["name"].as_str().unwrap_or("").to_string(), + arguments: b["input"].as_object().cloned().unwrap_or_default(), + }), + _ => {} + } + } + // An assistant turn needs visible content when it is replayed. + if !blocks + .iter() + .any(|b| matches!(b["type"].as_str(), Some("text" | "tool_use"))) + { + blocks.push(json!({"type": "text", "text": "(empty response)"})); + } + + let stop_reason = match self.stop_reason.as_deref() { + Some("end_turn") => StopReason::EndTurn, + Some("tool_use") => StopReason::ToolUse, + Some("max_tokens") => StopReason::MaxTokens, + Some("refusal") => StopReason::Refusal, + other => StopReason::Other(other.unwrap_or("unknown").to_string()), + }; + Ok(ChatResponse { + model: self.model, + text: text.join("\n"), + reasoning: (!reasoning.is_empty()).then(|| reasoning.join("\n\n")), + tool_calls, + usage, + stop_reason, + raw: Some(RawContent { + provider: PROVIDER.to_string(), + blocks, + }), + cleared_tool_results: self.cleared_tool_results, + }) + } +} + +fn append(block: &mut Value, field: &str, piece: &Value) { + let add = piece.as_str().unwrap_or(""); + let current = block[field].as_str().unwrap_or("").to_string(); + block[field] = Value::String(current + add); +} + +#[cfg(test)] +mod tests; diff --git a/crates/provider/src/anthropic/tests.rs b/crates/provider/src/anthropic/tests.rs new file mode 100644 index 00000000..0ed62fca --- /dev/null +++ b/crates/provider/src/anthropic/tests.rs @@ -0,0 +1,428 @@ +use super::*; +use crate::{CachePlan, ToolSpec}; +use gitagent_loader::Effort; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +fn cfg() -> AnthropicCfg { + AnthropicCfg { + thinking_display: ThinkingDisplay::Summarized, + thinking_binding: ThinkingBinding::Error, + refusal_fallback: true, + } +} + +fn request(messages: Vec) -> ChatRequest { + ChatRequest { + model: "claude-opus-5-5".into(), + system: "SYSTEM".into(), + messages, + tools: vec![ToolSpec { + name: "exec".into(), + description: "run".into(), + input_schema: json!({"type": "object"}), + }], + effort: Effort::High, + temperature: None, + max_output_tokens: 64000, + cache: CachePlan { + system: true, + last_messages: 2, + one_hour: false, + }, + server_masking: None, + } +} + +fn assistant_with_thinking() -> Message { + let input = json!({"cmd": "ls"}).as_object().unwrap().clone(); + Message { + role: Role::Assistant, + content: vec![Content::ToolUse { + id: "toolu_1".into(), + name: "exec".into(), + input: input.clone(), + }], + raw: Some(RawContent { + provider: PROVIDER.into(), + blocks: vec![ + json!({"type": "thinking", "thinking": "plan", "signature": "SIG"}), + json!({"type": "tool_use", "id": "toolu_1", "name": "exec", "input": input}), + ], + }), + } +} + +fn tool_result() -> Message { + Message::tool_results(vec![Content::ToolResult { + tool_use_id: "toolu_1".into(), + content: "a\nb".into(), + is_error: false, + images: vec![], + }]) +} + +#[test] +fn request_body_replays_raw_blocks_and_places_breakpoints() { + let req = request(vec![ + Message::user_text("task"), + assistant_with_thinking(), + tool_result(), + ]); + let body = request_body(&req, &cfg()); + + assert_eq!(body["stream"], true); + assert_eq!(body["max_tokens"], 64000); + assert_eq!(body["output_config"]["effort"], "high"); + assert_eq!(body["thinking"]["type"], "adaptive"); + assert_eq!(body["thinking"]["display"], "summarized"); + assert_eq!( + body["thinking"]["block_binding"]["prefix_mismatch_behavior"], + "error" + ); + assert_eq!(body["fallbacks"], "default"); + assert!(body.get("temperature").is_none()); + assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral"); + assert_eq!(body["tools"][0]["name"], "exec"); + + let msgs = body["messages"].as_array().unwrap(); + // First message is outside the last two: no breakpoint. + assert!(msgs[0]["content"][0].get("cache_control").is_none()); + // The thinking block is replayed exactly and never marked. + assert_eq!( + msgs[1]["content"][0], + json!({"type": "thinking", "thinking": "plan", "signature": "SIG"}) + ); + assert_eq!(msgs[1]["content"][1]["cache_control"]["type"], "ephemeral"); + assert_eq!(msgs[2]["content"][0]["type"], "tool_result"); + assert_eq!(msgs[2]["content"][0]["cache_control"]["type"], "ephemeral"); + assert!(msgs[2]["content"][0].get("is_error").is_none()); +} + +#[test] +fn shared_prefix_bytes_are_stable_between_turns() { + // Turn N's messages must reappear byte-for-byte in turn N+1, apart from + // the moving cache markers. + let strip = |mut v: Value| { + for m in v["messages"].as_array_mut().unwrap() { + for b in m["content"].as_array_mut().unwrap() { + b.as_object_mut().unwrap().remove("cache_control"); + } + } + v + }; + let turn1 = strip(request_body( + &request(vec![Message::user_text("task"), assistant_with_thinking()]), + &cfg(), + )); + let turn2 = strip(request_body( + &request(vec![ + Message::user_text("task"), + assistant_with_thinking(), + tool_result(), + ]), + &cfg(), + )); + let m1 = turn1["messages"].as_array().unwrap(); + let m2 = turn2["messages"].as_array().unwrap(); + assert_eq!(&m2[..m1.len()], &m1[..]); + assert_eq!(turn1["system"], turn2["system"]); + assert_eq!(turn1["tools"], turn2["tools"]); +} + +#[test] +fn optional_features_can_be_switched_off() { + let mut c = cfg(); + c.refusal_fallback = false; + c.thinking_display = ThinkingDisplay::Omitted; + let mut req = request(vec![Message::user_text("x")]); + req.temperature = Some(0.5); + req.cache = CachePlan::default(); + let body = request_body(&req, &c); + assert!(body.get("fallbacks").is_none()); + assert!(body["thinking"].get("display").is_none()); + assert_eq!(body["temperature"].to_string(), "0.5"); + assert!(body["system"][0].get("cache_control").is_none()); + assert_eq!(betas(&c), vec![BETA_THINKING_BINDING]); + assert_eq!(betas(&cfg()), vec![BETA_THINKING_BINDING, BETA_FALLBACK]); +} + +const STREAM: &str = "event: message_start\r\n\ +data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"model\":\"claude-opus-5-5\",\"usage\":{\"input_tokens\":10,\"cache_creation_input_tokens\":200,\"cache_read_input_tokens\":3000,\"output_tokens\":1}}}\r\n\r\n\ +event: content_block_start\n\ +data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}\n\n\ +event: content_block_delta\n\ +data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Look at \"}}\n\n\ +event: content_block_delta\n\ +data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"files.\"}}\n\n\ +event: content_block_delta\n\ +data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"SIG==\"}}\n\n\ +event: content_block_stop\n\ +data: {\"type\":\"content_block_stop\",\"index\":0}\n\n\ +event: ping\n\ +data: {\"type\":\"ping\"}\n\n\ +event: content_block_start\n\ +data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n\ +event: content_block_delta\n\ +data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Listing.\"}}\n\n\ +event: content_block_stop\n\ +data: {\"type\":\"content_block_stop\",\"index\":1}\n\n\ +event: content_block_start\n\ +data: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_9\",\"name\":\"exec\",\"input\":{}}}\n\n\ +event: content_block_delta\n\ +data: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"cmd\\\": \\\"ls \"}}\n\n\ +event: content_block_delta\n\ +data: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"-la\\\"}\"}}\n\n\ +event: content_block_stop\n\ +data: {\"type\":\"content_block_stop\",\"index\":2}\n\n\ +event: message_delta\n\ +data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"},\"usage\":{\"output_tokens\":57}}\n\n\ +event: message_stop\n\ +data: {\"type\":\"message_stop\"}\n\n"; + +fn parse_in_chunks(stream: &str, size: usize) -> Result { + let mut p = StreamParser::default(); + for chunk in stream.as_bytes().chunks(size) { + p.feed(chunk)?; + } + p.finish() +} + +#[test] +fn assembles_a_streamed_response_from_any_chunking() { + for size in [1, 7, 64, 4096] { + let r = parse_in_chunks(STREAM, size).unwrap(); + assert_eq!(r.model.as_deref(), Some("claude-opus-5-5")); + assert_eq!(r.text, "Listing."); + assert_eq!(r.reasoning.as_deref(), Some("Look at files.")); + assert_eq!(r.stop_reason, StopReason::ToolUse); + assert_eq!(r.tool_calls.len(), 1); + assert_eq!(r.tool_calls[0].id, "toolu_9"); + assert_eq!(r.tool_calls[0].arguments["cmd"], "ls -la"); + assert_eq!( + r.usage, + Usage { + input_tokens: 3210, + output_tokens: 57, + cache_read_tokens: 3000, + cache_write_tokens: 200, + } + ); + let raw = r.raw.unwrap(); + assert_eq!(raw.provider, PROVIDER); + assert_eq!( + raw.blocks[0], + json!({"type": "thinking", "thinking": "Look at files.", "signature": "SIG=="}) + ); + assert_eq!(raw.blocks[2]["input"], json!({"cmd": "ls -la"})); + } +} + +#[test] +fn incomplete_streams_and_stream_errors_are_retryable() { + let cut = &STREAM[..STREAM.find("event: message_delta").unwrap()]; + assert!(matches!( + parse_in_chunks(cut, 64), + Err(ProviderError::Transient(_)) + )); + let overloaded = "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n"; + assert!(matches!( + parse_in_chunks(overloaded, 64), + Err(ProviderError::Transient(_)) + )); +} + +#[test] +fn truncated_tool_input_and_empty_turns_are_repaired() { + let stream = "data: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\"usage\":{\"input_tokens\":5}}}\n\n\ +data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"t\",\"name\":\"exec\",\"input\":{}}}\n\n\ +data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"cmd\\\": \\\"cat < 1000000 maximum" + ) + ), + ProviderError::ContextOverflow(_) + )); + let fatal = classify_http_error(401, &body("authentication_error", "invalid x-api-key")); + assert!(matches!(&fatal, ProviderError::Fatal(m) if m.contains("401"))); + assert!(matches!( + classify_http_error(502, "bad gateway"), + ProviderError::Transient(_) + )); +} + +/// Serves one HTTP response and returns the raw request it received. +async fn serve_once( + status: &'static str, + content_type: &'static str, + body: String, +) -> (String, tokio::task::JoinHandle) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let handle = tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut req = Vec::new(); + let mut buf = [0u8; 8192]; + loop { + let n = sock.read(&mut buf).await.unwrap(); + req.extend_from_slice(&buf[..n]); + let text = String::from_utf8_lossy(&req); + if let Some(head_end) = text.find("\r\n\r\n") { + let len = text[..head_end] + .lines() + .find_map(|l| { + l.to_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().parse::().unwrap()) + }) + .unwrap_or(0); + if req.len() >= head_end + 4 + len { + break; + } + } + if n == 0 { + break; + } + } + let resp = format!( + "HTTP/1.1 {status}\r\ncontent-type: {content_type}\r\nconnection: close\r\n\r\n{body}" + ); + sock.write_all(resp.as_bytes()).await.unwrap(); + sock.shutdown().await.unwrap(); + String::from_utf8_lossy(&req).into_owned() + }); + (url, handle) +} + +#[tokio::test] +async fn completes_against_a_local_server() { + let (url, server) = serve_once("200 OK", "text/event-stream", STREAM.to_string()).await; + let provider = AnthropicProvider::new("test-key".into(), Some(url), cfg()).unwrap(); + let resp = provider + .complete(&request(vec![Message::user_text("task")])) + .await + .unwrap(); + assert_eq!(resp.tool_calls[0].arguments["cmd"], "ls -la"); + + let raw = server.await.unwrap(); + let lower = raw.to_lowercase(); + assert!(raw.starts_with("POST /v1/messages HTTP/1.1")); + assert!(lower.contains("x-api-key: test-key")); + assert!(lower.contains("anthropic-version: 2023-06-01")); + assert!(lower.contains( + "anthropic-beta: thinking-binding-controls-2026-08-01,server-side-fallback-2026-07-01" + )); + let body: Value = serde_json::from_str(&raw[raw.find("\r\n\r\n").unwrap() + 4..]).unwrap(); + assert_eq!(body["model"], "claude-opus-5-5"); + assert_eq!(body["messages"][0]["content"][0]["text"], "task"); +} + +#[tokio::test] +async fn http_errors_are_classified_without_leaking_the_key() { + let body = json!({"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}}).to_string(); + let (url, server) = serve_once("401 Unauthorized", "application/json", body).await; + let provider = AnthropicProvider::new("secret-key-123".into(), Some(url), cfg()).unwrap(); + let err = provider + .complete(&request(vec![Message::user_text("task")])) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(matches!(&err, ProviderError::Fatal(_))); + assert!(!err.to_string().contains("secret-key-123")); +} + +#[test] +fn tool_results_with_images_become_content_arrays() { + let msg = Message::tool_results(vec![Content::ToolResult { + tool_use_id: "t".into(), + content: "screenshot".into(), + is_error: false, + images: vec![crate::ImageData { + media_type: "image/png".into(), + data: "AAAA".into(), + }], + }]); + let blocks = message_blocks(&msg); + assert_eq!( + blocks[0]["content"], + json!([ + {"type": "text", "text": "screenshot"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}} + ]) + ); +} + +#[test] +fn server_masking_and_one_hour_cache_are_requested() { + let mut req = request(vec![Message::user_text("task")]); + req.cache.one_hour = true; + req.server_masking = Some(crate::ServerMasking { + trigger_tokens: 50000, + keep: 8, + clear_at_least: 25000, + }); + let body = request_body(&req, &cfg()); + assert_eq!( + body["system"][0]["cache_control"], + json!({"type": "ephemeral", "ttl": "1h"}) + ); + assert_eq!( + body["messages"][0]["content"][0]["cache_control"]["ttl"], + "1h" + ); + assert_eq!( + body["context_management"]["edits"][0], + json!({ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": 50000}, + "keep": {"type": "tool_uses", "value": 8}, + "clear_at_least": {"type": "input_tokens", "value": 25000} + }) + ); + // Without masking there is no context_management field. + let plain = request_body(&request(vec![Message::user_text("task")]), &cfg()); + assert!(plain.get("context_management").is_none()); +} + +#[test] +fn reports_tool_results_cleared_by_the_server() { + let stream = STREAM.replace( + "\"delta\":{\"stop_reason\":\"tool_use\"},", + "\"delta\":{\"stop_reason\":\"tool_use\"},\"context_management\":{\"applied_edits\":[{\"type\":\"clear_tool_uses_20250919\",\"cleared_tool_uses\":6,\"cleared_input_tokens\":41000}]},", + ); + let r = parse_in_chunks(&stream, 64).unwrap(); + assert_eq!(r.cleared_tool_results, 6); +} diff --git a/crates/provider/src/compat.rs b/crates/provider/src/compat.rs new file mode 100644 index 00000000..3a20edf8 --- /dev/null +++ b/crates/provider/src/compat.rs @@ -0,0 +1,306 @@ +//! OpenAI-compatible Chat Completions adapter, for open-weight models served +//! by vLLM, SGLang, Ollama, OpenRouter and similar endpoints. Profiles for +//! these models usually also turn on the text fallback for tool calls. + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use gitagent_loader::Family; +use serde_json::{json, Value}; + +use crate::sse::SseDecoder; +use crate::{ + ChatRequest, ChatResponse, Content, Message, Provider, ProviderError, Role, StopReason, + ToolCall, Usage, +}; + +pub struct CompatProvider { + client: reqwest::Client, + base_url: String, + api_key: Option, +} + +/// Well-known endpoints by provider name; anything else needs `{NAME}_BASE_URL`. +const KNOWN: &[(&str, &str, &str)] = &[ + ( + "openrouter", + "https://openrouter.ai/api/v1", + "OPENROUTER_API_KEY", + ), + ("ollama", "http://localhost:11434/v1", "OLLAMA_API_KEY"), + ("vllm", "http://localhost:8000/v1", "VLLM_API_KEY"), +]; + +impl CompatProvider { + pub fn new(base_url: String, api_key: Option) -> anyhow::Result { + Ok(CompatProvider { + client: crate::http_client()?, + base_url: base_url.trim_end_matches('/').to_string(), + api_key, + }) + } + + /// For a model given as `:`: the base URL comes from + /// `{NAME}_BASE_URL` (or a built-in default for openrouter, ollama and + /// vllm) and the key from `{NAME}_API_KEY` or `GITAGENT_API_KEY`. + pub fn from_env(name: &str) -> anyhow::Result { + let upper = name.to_uppercase().replace('-', "_"); + let known = KNOWN.iter().find(|(n, _, _)| *n == name); + let base = std::env::var(format!("{upper}_BASE_URL")) + .ok() + .or_else(|| known.map(|(_, url, _)| url.to_string())) + .ok_or_else(|| { + anyhow::anyhow!( + "set {upper}_BASE_URL to the OpenAI-compatible endpoint for {name:?}" + ) + })?; + let key_var = known.map_or(format!("{upper}_API_KEY"), |(_, _, k)| k.to_string()); + let key = [key_var.as_str(), "GITAGENT_API_KEY"] + .iter() + .find_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty())); + Self::new(base, key) + } +} + +#[async_trait] +impl Provider for CompatProvider { + async fn complete(&self, req: &ChatRequest) -> Result { + let body = serde_json::to_vec(&request_body(req)) + .map_err(|e| ProviderError::Fatal(e.to_string()))?; + let mut http = self + .client + .post(format!("{}/chat/completions", self.base_url)) + .header("content-type", "application/json"); + if let Some(key) = &self.api_key { + http = http.bearer_auth(key); + } + let mut resp = http + .body(body) + .send() + .await + .map_err(|e| ProviderError::Transient(format!("request failed: {e}")))?; + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + return Err(crate::openai::classify_http_error(status.as_u16(), &text)); + } + let mut stream = StreamParser::default(); + while let Some(chunk) = resp + .chunk() + .await + .map_err(|e| ProviderError::Transient(format!("stream interrupted: {e}")))? + { + stream.feed(&chunk)?; + } + stream.finish() + } + + fn family(&self) -> Family { + Family::Compat + } +} + +pub(crate) fn request_body(req: &ChatRequest) -> Value { + let mut messages = vec![json!({"role": "system", "content": req.system})]; + for m in &req.messages { + messages.extend(convert(m)); + } + let tools: Vec = req + .tools + .iter() + .map(|t| { + json!({"type": "function", "function": { + "name": t.name, "description": t.description, "parameters": t.input_schema + }}) + }) + .collect(); + let mut body = json!({ + "model": req.model, + "messages": messages, + "tools": tools, + "tool_choice": "auto", + "max_tokens": req.max_output_tokens, + "stream": true, + "stream_options": {"include_usage": true}, + }); + if let Some(t) = req.temperature { + body["temperature"] = crate::temperature_json(t); + } + body +} + +fn convert(m: &Message) -> Vec { + let text: Vec<&str> = m + .content + .iter() + .filter_map(|c| match c { + Content::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect(); + match m.role { + Role::Assistant => { + let calls: Vec = m + .content + .iter() + .filter_map(|c| match c { + Content::ToolUse { id, name, input } => Some(json!({ + "id": id, "type": "function", + "function": {"name": name, "arguments": Value::Object(input.clone()).to_string()} + })), + _ => None, + }) + .collect(); + let mut msg = json!({"role": "assistant", "content": text.join("\n")}); + if !calls.is_empty() { + msg["tool_calls"] = Value::Array(calls); + } + vec![msg] + } + Role::User => { + let mut out: Vec = m + .content + .iter() + .filter_map(|c| match c { + Content::ToolResult { + tool_use_id, + content, + images, + .. + } => { + let note = if images.is_empty() { + String::new() + } else { + "\n[images are not supported by this endpoint and were left out]".into() + }; + Some(json!({"role": "tool", "tool_call_id": tool_use_id, "content": format!("{content}{note}")})) + } + _ => None, + }) + .collect(); + if !text.is_empty() { + out.push(json!({"role": "user", "content": text.join("\n")})); + } + out + } + } +} + +#[derive(Default)] +pub(crate) struct StreamParser { + sse: SseDecoder, + text: String, + reasoning: String, + /// Tool calls by stream index: id, name, accumulated arguments. + calls: BTreeMap, + finish: Option, + usage: Value, + model: Option, + done: bool, +} + +impl StreamParser { + pub fn feed(&mut self, bytes: &[u8]) -> Result<(), ProviderError> { + for data in self.sse.feed(bytes) { + if data.trim() == "[DONE]" { + self.done = true; + continue; + } + let ev: Value = serde_json::from_str(&data) + .map_err(|e| ProviderError::Transient(format!("bad stream event: {e}")))?; + if ev.get("error").is_some() { + return Err(crate::openai::classify_http_error(0, &data)); + } + if let Some(m) = ev["model"].as_str() { + self.model = Some(m.to_string()); + } + if ev["usage"].is_object() { + self.usage = ev["usage"].clone(); + } + let Some(choice) = ev["choices"].get(0) else { + continue; + }; + let delta = &choice["delta"]; + if let Some(t) = delta["content"].as_str() { + self.text.push_str(t); + } + // vLLM/DeepSeek use reasoning_content, OpenRouter uses reasoning. + for key in ["reasoning_content", "reasoning"] { + if let Some(t) = delta[key].as_str() { + self.reasoning.push_str(t); + } + } + for tc in delta["tool_calls"].as_array().into_iter().flatten() { + let index = tc["index"].as_u64().unwrap_or(self.calls.len() as u64); + let entry = self.calls.entry(index).or_default(); + if let Some(id) = tc["id"].as_str() { + entry.0 = id.to_string(); + } + if let Some(name) = tc["function"]["name"].as_str() { + entry.1.push_str(name); + } + if let Some(args) = tc["function"]["arguments"].as_str() { + entry.2.push_str(args); + } + } + if let Some(f) = choice["finish_reason"].as_str() { + self.finish = Some(f.to_string()); + } + } + Ok(()) + } + + pub fn finish(self) -> Result { + if self.finish.is_none() && !self.done { + return Err(ProviderError::Transient( + "stream ended before the response finished".into(), + )); + } + let tool_calls: Vec = self + .calls + .into_values() + .enumerate() + .map(|(i, (id, name, args))| ToolCall { + id: if id.is_empty() { + format!("call_{}", i + 1) + } else { + id + }, + name, + arguments: serde_json::from_str::(&args) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(), + }) + .collect(); + let count = |v: &Value| v.as_u64().unwrap_or(0); + let usage = Usage { + input_tokens: count(&self.usage["prompt_tokens"]), + output_tokens: count(&self.usage["completion_tokens"]), + cache_read_tokens: count(&self.usage["prompt_tokens_details"]["cached_tokens"]), + cache_write_tokens: 0, + }; + let stop_reason = match self.finish.as_deref() { + _ if !tool_calls.is_empty() => StopReason::ToolUse, + Some("stop") | None => StopReason::EndTurn, + Some("length") => StopReason::MaxTokens, + Some("content_filter") => StopReason::Refusal, + Some(other) => StopReason::Other(other.to_string()), + }; + Ok(ChatResponse { + model: self.model, + text: self.text, + reasoning: (!self.reasoning.is_empty()).then_some(self.reasoning), + tool_calls, + usage, + stop_reason, + // The history is rebuilt from text and calls; reasoning is not + // sent back, as these endpoints expect. + raw: None, + cleared_tool_results: 0, + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provider/src/compat/ollama-stream.txt b/crates/provider/src/compat/ollama-stream.txt new file mode 100644 index 00000000..ef80ee29 --- /dev/null +++ b/crates/provider/src/compat/ollama-stream.txt @@ -0,0 +1,8 @@ +data: {"id":"chatcmpl-554","object":"chat.completion.chunk","created":1792011154,"model":"qwen2.5:1.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"id":"call_4w9n9qm8","index":0,"type":"function","function":{"name":"exec","arguments":"{\"cmd\":\"cat hi.txt\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-554","object":"chat.completion.chunk","created":1792011154,"model":"qwen2.5:1.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-554","object":"chat.completion.chunk","created":1792011154,"model":"qwen2.5:1.5b","system_fingerprint":"fp_ollama","choices":[],"usage":{"prompt_tokens":1417,"prompt_tokens_details":{"cached_tokens":1416},"completion_tokens":21,"total_tokens":1438},"timings":{"prompt_n":1417,"prompt_ms":24,"prompt_per_token_ms":0.016937191249117856,"prompt_per_second":59041.666666666664,"predicted_n":21,"predicted_ms":247,"predicted_per_token_ms":11.761904761904763,"predicted_per_second":85.02024291497976}} + +data: [DONE] + diff --git a/crates/provider/src/compat/tests.rs b/crates/provider/src/compat/tests.rs new file mode 100644 index 00000000..d32f5bb7 --- /dev/null +++ b/crates/provider/src/compat/tests.rs @@ -0,0 +1,135 @@ +use super::*; +use crate::test_server::{body_of, serve_once}; +use crate::{CachePlan, ToolSpec}; +use gitagent_loader::Effort; + +fn request(messages: Vec) -> ChatRequest { + ChatRequest { + model: "qwen-test".into(), + system: "SYSTEM".into(), + messages, + tools: vec![ToolSpec { + name: "exec".into(), + description: "run".into(), + input_schema: json!({"type": "object"}), + }], + effort: Effort::Medium, + temperature: Some(0.7), + max_output_tokens: 16000, + cache: CachePlan::default(), + server_masking: None, + } +} + +fn data(v: Value) -> String { + format!("data: {v}\n\n") +} + +fn stream() -> String { + [ + data(json!({"model": "qwen-test", "choices": [{"delta": {"role": "assistant", "reasoning_content": "Think."}}]})), + data(json!({"choices": [{"delta": {"content": "Run it."}}]})), + data(json!({"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "c1", "type": "function", "function": {"name": "exec", "arguments": "{\"cmd\": "}}]}}]})), + data(json!({"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": "\"ls\"}"}}]}, "finish_reason": "tool_calls"}]})), + data(json!({"choices": [], "usage": {"prompt_tokens": 700, "completion_tokens": 30, "prompt_tokens_details": {"cached_tokens": 600}}})), + "data: [DONE]\n\n".to_string(), + ] + .concat() +} + +fn parse(s: &str, chunk: usize) -> Result { + let mut p = StreamParser::default(); + for c in s.as_bytes().chunks(chunk) { + p.feed(c)?; + } + p.finish() +} + +#[test] +fn assembles_streamed_text_reasoning_and_tool_calls() { + for size in [1, 23, 4096] { + let r = parse(&stream(), size).unwrap(); + assert_eq!(r.text, "Run it."); + assert_eq!(r.reasoning.as_deref(), Some("Think.")); + assert_eq!(r.stop_reason, StopReason::ToolUse); + assert_eq!(r.tool_calls[0].id, "c1"); + assert_eq!(r.tool_calls[0].arguments["cmd"], "ls"); + assert_eq!(r.usage.input_tokens, 700); + assert_eq!(r.usage.cache_read_tokens, 600); + assert!(r.raw.is_none()); + } +} + +#[test] +fn request_uses_system_tool_and_tool_result_messages() { + let r = parse(&stream(), 4096).unwrap(); + let history = vec![ + Message::user_text("task"), + r.to_message(), + Message::tool_results(vec![ + Content::ToolResult { + tool_use_id: "c1".into(), + content: "a.txt".into(), + is_error: false, + images: vec![], + }, + Content::Text { + text: "Time check".into(), + }, + ]), + ]; + let body = request_body(&request(history)); + let msgs = body["messages"].as_array().unwrap(); + assert_eq!(msgs[0], json!({"role": "system", "content": "SYSTEM"})); + assert_eq!(msgs[1], json!({"role": "user", "content": "task"})); + assert_eq!( + msgs[2]["tool_calls"][0]["function"]["arguments"], + "{\"cmd\":\"ls\"}" + ); + assert_eq!( + msgs[3], + json!({"role": "tool", "tool_call_id": "c1", "content": "a.txt"}) + ); + assert_eq!(msgs[4], json!({"role": "user", "content": "Time check"})); + assert_eq!(body["temperature"], 0.7); + assert_eq!(body["stream_options"]["include_usage"], true); + assert_eq!(body["tools"][0]["function"]["name"], "exec"); +} + +#[test] +fn text_only_and_truncated_replies() { + let s = [ + data(json!({"choices": [{"delta": {"content": "{}"}, "finish_reason": "stop"}]})), + "data: [DONE]\n\n".to_string(), + ] + .concat(); + let r = parse(&s, 64).unwrap(); + assert_eq!(r.stop_reason, StopReason::EndTurn); + assert!(r.tool_calls.is_empty()); + let s = data(json!({"choices": [{"delta": {"content": "x"}, "finish_reason": "length"}]})); + assert_eq!(parse(&s, 64).unwrap().stop_reason, StopReason::MaxTokens); + let cut = data(json!({"choices": [{"delta": {"content": "x"}}]})); + assert!(matches!(parse(&cut, 64), Err(ProviderError::Transient(_)))); +} + +#[tokio::test] +async fn completes_against_a_local_server_without_a_key() { + let (url, server) = serve_once("200 OK", "text/event-stream", stream()).await; + let p = CompatProvider::new(format!("{url}/v1"), None).unwrap(); + let r = p + .complete(&request(vec![Message::user_text("task")])) + .await + .unwrap(); + assert_eq!(r.tool_calls.len(), 1); + let raw = server.await.unwrap(); + assert!(raw.starts_with("POST /v1/chat/completions HTTP/1.1")); + assert!(!raw.to_lowercase().contains("authorization:")); + assert_eq!(body_of(&raw)["model"], "qwen-test"); +} + +#[test] +fn parses_a_recorded_ollama_stream() { + let r = parse(include_str!("ollama-stream.txt"), 4096).unwrap(); + assert_eq!(r.tool_calls.len(), 1, "{r:?}"); + assert_eq!(r.tool_calls[0].name, "exec"); +} diff --git a/crates/provider/src/google.rs b/crates/provider/src/google.rs new file mode 100644 index 00000000..7d959e14 --- /dev/null +++ b/crates/provider/src/google.rs @@ -0,0 +1,335 @@ +//! Google Gemini adapter (`streamGenerateContent`, server-sent events). +//! +//! Model turns keep Gemini's own parts (`Message::raw`) and are replayed +//! unchanged: parts can carry a `thoughtSignature` that must come back with +//! them, or the request fails. + +use std::collections::HashMap; + +use async_trait::async_trait; +use gitagent_loader::{Effort, Family}; +use serde_json::{json, Map, Value}; + +use crate::sse::SseDecoder; +use crate::{ + ChatRequest, ChatResponse, Content, Message, Provider, ProviderError, RawContent, Role, + StopReason, ToolCall, Usage, +}; + +pub const PROVIDER: &str = "google"; +const DEFAULT_BASE_URL: &str = "https://generativelanguage.googleapis.com"; + +pub struct GoogleProvider { + client: reqwest::Client, + base_url: String, + api_key: String, +} + +impl GoogleProvider { + pub fn new(api_key: String, base_url: Option) -> anyhow::Result { + Ok(GoogleProvider { + client: crate::http_client()?, + base_url: base_url + .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) + .trim_end_matches('/') + .to_string(), + api_key, + }) + } + + /// Reads `GEMINI_API_KEY` (or `GOOGLE_API_KEY`) and, if set, `GEMINI_BASE_URL`. + pub fn from_env() -> anyhow::Result { + let key = ["GEMINI_API_KEY", "GOOGLE_API_KEY"] + .iter() + .find_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty())) + .ok_or_else(|| anyhow::anyhow!("GEMINI_API_KEY is not set"))?; + Self::new(key, std::env::var("GEMINI_BASE_URL").ok()) + } +} + +#[async_trait] +impl Provider for GoogleProvider { + async fn complete(&self, req: &ChatRequest) -> Result { + let body = serde_json::to_vec(&request_body(req)) + .map_err(|e| ProviderError::Fatal(e.to_string()))?; + let url = format!( + "{}/v1beta/models/{}:streamGenerateContent?alt=sse", + self.base_url, req.model + ); + let mut resp = self + .client + .post(url) + .header("x-goog-api-key", &self.api_key) + .header("content-type", "application/json") + .body(body) + .send() + .await + .map_err(|e| ProviderError::Transient(format!("request failed: {e}")))?; + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + return Err(classify_http_error(status.as_u16(), &text)); + } + let mut stream = StreamParser::default(); + while let Some(chunk) = resp + .chunk() + .await + .map_err(|e| ProviderError::Transient(format!("stream interrupted: {e}")))? + { + stream.feed(&chunk)?; + } + stream.finish() + } + + fn family(&self) -> Family { + Family::Google + } +} + +pub(crate) fn request_body(req: &ChatRequest) -> Value { + let declarations: Vec = req + .tools + .iter() + .map( + |t| json!({"name": t.name, "description": t.description, "parameters": t.input_schema}), + ) + .collect(); + let level = match req.effort { + Effort::Low => "low", + Effort::Medium => "medium", + Effort::High | Effort::XHigh | Effort::Max => "high", + }; + let mut generation = json!({ + "maxOutputTokens": req.max_output_tokens, + "thinkingConfig": {"thinkingLevel": level, "includeThoughts": true}, + }); + if let Some(t) = req.temperature { + generation["temperature"] = crate::temperature_json(t); + } + json!({ + "systemInstruction": {"parts": [{"text": req.system}]}, + "contents": contents(&req.messages), + "tools": [{"functionDeclarations": declarations}], + "toolConfig": {"functionCallingConfig": {"mode": "AUTO"}}, + "generationConfig": generation, + }) +} + +/// Converts the history. Function responses need the function's name, which +/// is looked up from the call with the same id. +fn contents(messages: &[Message]) -> Vec { + // Our call id -> (function name, Gemini's own id if it gave one). + let mut calls: HashMap)> = HashMap::new(); + let mut out = Vec::new(); + for m in messages { + match m.role { + Role::Assistant => { + let parts = match m.raw.as_ref().filter(|r| r.provider == PROVIDER) { + Some(raw) => raw.blocks.clone(), + None => assistant_parts(m), + }; + // Map ids for the responses that follow. + let mut gemini_ids = parts + .iter() + .filter(|p| p.get("functionCall").is_some()) + .map(|p| p["functionCall"]["id"].as_str().map(str::to_string)); + for c in &m.content { + if let Content::ToolUse { id, name, .. } = c { + let gid = gemini_ids.next().flatten(); + calls.insert(id.clone(), (name.clone(), gid)); + } + } + out.push(json!({"role": "model", "parts": parts})); + } + Role::User => { + let mut parts = Vec::new(); + for c in &m.content { + match c { + Content::Text { text } => parts.push(json!({"text": text})), + Content::ToolResult { + tool_use_id, + content, + is_error, + images, + } => { + let (name, gid) = calls + .get(tool_use_id) + .cloned() + .unwrap_or_else(|| (String::from("unknown"), None)); + let key = if *is_error { "error" } else { "result" }; + let mut response = json!({"name": name, "response": {key: content}}); + if let Some(gid) = gid { + response["id"] = json!(gid); + } + parts.push(json!({"functionResponse": response})); + parts.extend(images.iter().map(|i| { + json!({"inlineData": {"mimeType": i.media_type, "data": i.data}}) + })); + } + Content::ToolUse { .. } => {} + } + } + out.push(json!({"role": "user", "parts": parts})); + } + } + } + out +} + +fn assistant_parts(m: &Message) -> Vec { + m.content + .iter() + .filter_map(|c| match c { + Content::Text { text } => Some(json!({"text": text})), + Content::ToolUse { name, input, .. } => { + Some(json!({"functionCall": {"name": name, "args": input}})) + } + Content::ToolResult { .. } => None, + }) + .collect() +} + +pub(crate) fn classify_http_error(status: u16, body: &str) -> ProviderError { + let parsed: Value = serde_json::from_str(body).unwrap_or(Value::Null); + // Errors come as an object or as a one-element array of objects. + let err = if parsed.is_array() { + &parsed[0]["error"] + } else { + &parsed["error"] + }; + let state = err["status"].as_str().unwrap_or(""); + let message = err["message"] + .as_str() + .map(str::to_string) + .unwrap_or_else(|| body.chars().take(500).collect()); + let detail = format!("HTTP {status} {state}: {message}"); + let lower = message.to_lowercase(); + if lower.contains("exceeds the maximum number of tokens") || lower.contains("too long") { + ProviderError::ContextOverflow(detail) + } else if matches!(status, 408 | 429 | 500 | 502 | 503 | 504) + || matches!( + state, + "RESOURCE_EXHAUSTED" | "UNAVAILABLE" | "INTERNAL" | "DEADLINE_EXCEEDED" + ) + { + ProviderError::Transient(detail) + } else { + ProviderError::Fatal(detail) + } +} + +#[derive(Default)] +pub(crate) struct StreamParser { + sse: SseDecoder, + parts: Vec, + finish: Option, + blocked: Option, + usage: Value, + model: Option, + events: usize, +} + +impl StreamParser { + pub fn feed(&mut self, bytes: &[u8]) -> Result<(), ProviderError> { + for data in self.sse.feed(bytes) { + let ev: Value = serde_json::from_str(&data) + .map_err(|e| ProviderError::Transient(format!("bad stream event: {e}")))?; + if ev.get("error").is_some() { + return Err(classify_http_error(0, &data)); + } + self.events += 1; + if let Some(reason) = ev["promptFeedback"]["blockReason"].as_str() { + self.blocked = Some(reason.to_string()); + } + if let Some(cand) = ev["candidates"].get(0) { + for p in cand["content"]["parts"].as_array().into_iter().flatten() { + self.parts.push(p.clone()); + } + if let Some(f) = cand["finishReason"].as_str() { + self.finish = Some(f.to_string()); + } + } + if ev["usageMetadata"].is_object() { + self.usage = ev["usageMetadata"].clone(); + } + if let Some(m) = ev["modelVersion"].as_str() { + self.model = Some(m.to_string()); + } + } + Ok(()) + } + + pub fn finish(self) -> Result { + if self.events == 0 || (self.finish.is_none() && self.blocked.is_none()) { + return Err(ProviderError::Transient( + "stream ended before the response finished".into(), + )); + } + let mut text = Vec::new(); + let mut reasoning = Vec::new(); + let mut tool_calls = Vec::new(); + for p in &self.parts { + if let Some(call) = p.get("functionCall") { + let n = tool_calls.len() + 1; + tool_calls.push(ToolCall { + // Gemini may omit ids; ours only need to be unique. + id: call["id"] + .as_str() + .map_or_else(|| format!("gemini_call_{n}"), str::to_string), + name: call["name"].as_str().unwrap_or("").to_string(), + arguments: call["args"].as_object().cloned().unwrap_or_else(Map::new), + }); + } else if let Some(t) = p["text"].as_str() { + if p["thought"].as_bool() == Some(true) { + reasoning.push(t.to_string()); + } else { + text.push(t.to_string()); + } + } + } + let mut parts = self.parts; + if !parts.iter().any(|p| { + p.get("functionCall").is_some() || (p.get("text").is_some() && p["thought"] != true) + }) { + parts.push(json!({"text": "(empty response)"})); + } + let count = |k: &str| self.usage[k].as_u64().unwrap_or(0); + let usage = Usage { + input_tokens: count("promptTokenCount"), + output_tokens: count("candidatesTokenCount") + count("thoughtsTokenCount"), + cache_read_tokens: count("cachedContentTokenCount"), + cache_write_tokens: 0, + }; + let refusal = matches!( + self.finish.as_deref(), + Some("SAFETY" | "PROHIBITED_CONTENT" | "BLOCKLIST" | "SPII" | "RECITATION") + ) || self.blocked.is_some(); + let stop_reason = if refusal { + StopReason::Refusal + } else if !tool_calls.is_empty() { + StopReason::ToolUse + } else { + match self.finish.as_deref() { + Some("STOP") => StopReason::EndTurn, + Some("MAX_TOKENS") => StopReason::MaxTokens, + other => StopReason::Other(other.unwrap_or("unknown").to_lowercase()), + } + }; + Ok(ChatResponse { + model: self.model, + text: text.concat(), + reasoning: (!reasoning.is_empty()).then(|| reasoning.concat()), + tool_calls, + usage, + stop_reason, + raw: Some(RawContent { + provider: PROVIDER.to_string(), + blocks: parts, + }), + cleared_tool_results: 0, + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provider/src/google/tests.rs b/crates/provider/src/google/tests.rs new file mode 100644 index 00000000..00cdc06f --- /dev/null +++ b/crates/provider/src/google/tests.rs @@ -0,0 +1,181 @@ +use super::*; +use crate::test_server::{body_of, serve_once}; +use crate::{CachePlan, ToolSpec}; + +fn request(messages: Vec) -> ChatRequest { + ChatRequest { + model: "gemini-test".into(), + system: "SYSTEM".into(), + messages, + tools: vec![ToolSpec { + name: "exec".into(), + description: "run".into(), + input_schema: json!({"type": "object", "properties": {"cmd": {"type": "string"}}}), + }], + effort: Effort::High, + temperature: None, + max_output_tokens: 32000, + cache: CachePlan::default(), + server_masking: None, + } +} + +fn data(v: Value) -> String { + format!("data: {v}\r\n\r\n") +} + +fn stream() -> String { + [ + data(json!({"candidates": [{"content": {"role": "model", "parts": [{"text": "Plan: list files.", "thought": true}]}}]})), + data(json!({"candidates": [{"content": {"role": "model", "parts": [{"text": "Listing."}]}}]})), + data(json!({"candidates": [{"content": {"role": "model", "parts": [ + {"functionCall": {"id": "g1", "name": "exec", "args": {"cmd": "ls"}}, "thoughtSignature": "SIG"} + ]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 900, "cachedContentTokenCount": 512, "candidatesTokenCount": 40, "thoughtsTokenCount": 60}, + "modelVersion": "gemini-test-001"})), + ] + .concat() +} + +fn parse(s: &str, chunk: usize) -> Result { + let mut p = StreamParser::default(); + for c in s.as_bytes().chunks(chunk) { + p.feed(c)?; + } + p.finish() +} + +#[test] +fn assembles_a_streamed_response_and_keeps_signatures() { + for size in [1, 17, 4096] { + let r = parse(&stream(), size).unwrap(); + assert_eq!(r.text, "Listing."); + assert_eq!(r.reasoning.as_deref(), Some("Plan: list files.")); + assert_eq!(r.stop_reason, StopReason::ToolUse); + assert_eq!(r.tool_calls[0].id, "g1"); + assert_eq!(r.tool_calls[0].arguments["cmd"], "ls"); + assert_eq!( + r.usage, + Usage { + input_tokens: 900, + output_tokens: 100, + cache_read_tokens: 512, + cache_write_tokens: 0 + } + ); + let raw = r.raw.unwrap(); + assert_eq!(raw.blocks[2]["thoughtSignature"], "SIG"); + assert_eq!(r.model.as_deref(), Some("gemini-test-001")); + } +} + +#[test] +fn request_replays_parts_and_names_function_responses() { + let r = parse(&stream(), 4096).unwrap(); + let history = vec![ + Message::user_text("task"), + r.to_message(), + Message::tool_results(vec![Content::ToolResult { + tool_use_id: "g1".into(), + content: "a.txt".into(), + is_error: false, + images: vec![], + }]), + ]; + let body = request_body(&request(history)); + assert_eq!(body["systemInstruction"]["parts"][0]["text"], "SYSTEM"); + assert_eq!( + body["generationConfig"]["thinkingConfig"]["thinkingLevel"], + "high" + ); + assert_eq!(body["generationConfig"]["maxOutputTokens"], 32000); + assert_eq!(body["tools"][0]["functionDeclarations"][0]["name"], "exec"); + let contents = body["contents"].as_array().unwrap(); + assert_eq!( + contents[0], + json!({"role": "user", "parts": [{"text": "task"}]}) + ); + assert_eq!(contents[1]["role"], "model"); + assert_eq!(contents[1]["parts"][2]["thoughtSignature"], "SIG"); + assert_eq!( + contents[2], + json!({"role": "user", "parts": [{"functionResponse": {"id": "g1", "name": "exec", "response": {"result": "a.txt"}}}]}) + ); +} + +#[test] +fn calls_without_ids_get_ids_and_responses_match_by_name() { + let s = data( + json!({"candidates": [{"content": {"role": "model", "parts": [ + {"functionCall": {"name": "exec", "args": {"cmd": "pwd"}}} + ]}, "finishReason": "STOP"}]}), + ); + let r = parse(&s, 64).unwrap(); + assert_eq!(r.tool_calls[0].id, "gemini_call_1"); + let history = vec![ + Message::user_text("t"), + r.to_message(), + Message::tool_results(vec![Content::ToolResult { + tool_use_id: "gemini_call_1".into(), + content: "/".into(), + is_error: true, + images: vec![], + }]), + ]; + let body = request_body(&request(history)); + assert_eq!( + body["contents"][2]["parts"][0], + json!({"functionResponse": {"name": "exec", "response": {"error": "/"}}}) + ); +} + +#[test] +fn safety_blocks_errors_and_truncation() { + let s = data(json!({"promptFeedback": {"blockReason": "SAFETY"}})); + assert_eq!(parse(&s, 64).unwrap().stop_reason, StopReason::Refusal); + let s = data( + json!({"candidates": [{"content": {"parts": [{"text": "partial"}]}, "finishReason": "MAX_TOKENS"}]}), + ); + assert_eq!(parse(&s, 64).unwrap().stop_reason, StopReason::MaxTokens); + let cut = data(json!({"candidates": [{"content": {"parts": [{"text": "partial"}]}}]})); + assert!(matches!(parse(&cut, 64), Err(ProviderError::Transient(_)))); + let err = |status: u16, state: &str, msg: &str| { + classify_http_error( + status, + &json!({"error": {"code": status, "status": state, "message": msg}}).to_string(), + ) + }; + assert!(matches!( + err(429, "RESOURCE_EXHAUSTED", "quota"), + ProviderError::Transient(_) + )); + assert!(matches!( + err( + 400, + "INVALID_ARGUMENT", + "The input token count exceeds the maximum number of tokens allowed" + ), + ProviderError::ContextOverflow(_) + )); + assert!(matches!( + err(403, "PERMISSION_DENIED", "bad key"), + ProviderError::Fatal(_) + )); +} + +#[tokio::test] +async fn completes_against_a_local_server() { + let (url, server) = serve_once("200 OK", "text/event-stream", stream()).await; + let p = GoogleProvider::new("g-key".into(), Some(url)).unwrap(); + let r = p + .complete(&request(vec![Message::user_text("task")])) + .await + .unwrap(); + assert_eq!(r.tool_calls[0].name, "exec"); + let raw = server.await.unwrap(); + assert!( + raw.starts_with("POST /v1beta/models/gemini-test:streamGenerateContent?alt=sse HTTP/1.1") + ); + assert!(raw.to_lowercase().contains("x-goog-api-key: g-key")); + assert_eq!(body_of(&raw)["contents"][0]["parts"][0]["text"], "task"); +} diff --git a/crates/provider/src/lib.rs b/crates/provider/src/lib.rs new file mode 100644 index 00000000..e9157689 --- /dev/null +++ b/crates/provider/src/lib.rs @@ -0,0 +1,83 @@ +//! Model provider interface and adapters. +//! +//! Adapters are thin hand-written HTTP clients so that the bytes sent to the +//! provider stay under our control (prompt-cache hits depend on it). + +pub mod anthropic; +pub mod compat; +pub mod google; +pub mod openai; +mod scripted; +mod sse; +#[cfg(test)] +mod test_server; +mod types; + +pub use anthropic::AnthropicProvider; +pub use compat::CompatProvider; +pub use google::GoogleProvider; +pub use openai::OpenAiProvider; +pub use scripted::ScriptedProvider; +pub use types::*; + +use async_trait::async_trait; +use gitagent_loader::Family; + +#[async_trait] +pub trait Provider: Send + Sync { + async fn complete(&self, req: &ChatRequest) -> Result; + fn family(&self) -> Family; +} + +#[derive(Debug, thiserror::Error)] +pub enum ProviderError { + /// Worth retrying after a backoff: rate limits, overload, 5xx, network errors. + #[error("transient provider error: {0}")] + Transient(String), + /// The request does not fit the model's context window. + #[error("context window exceeded: {0}")] + ContextOverflow(String), + /// Anything else; retrying will not help. + #[error("provider error: {0}")] + Fatal(String), +} + +/// The provider a loaded configuration asks for: `scripted`, `anthropic`, +/// `openai`, `google` / `gemini`, or any OpenAI-compatible endpoint by name. +pub fn for_spec(spec: &gitagent_loader::AgentSpec) -> anyhow::Result> { + use anyhow::Context; + match spec.model.provider.as_str() { + "scripted" => Ok(Box::new(ScriptedProvider::from_file( + std::path::Path::new(&spec.model.model), + )?)), + "anthropic" => { + let cfg = spec.profile.anthropic.clone().with_context(|| { + format!("profile {} has no `anthropic` section", spec.profile_source) + })?; + Ok(Box::new(AnthropicProvider::from_env(cfg)?)) + } + "openai" => { + let cfg = spec.profile.openai.clone().with_context(|| { + format!("profile {} has no `openai` section", spec.profile_source) + })?; + Ok(Box::new(OpenAiProvider::from_env(cfg)?)) + } + "google" | "gemini" => Ok(Box::new(GoogleProvider::from_env()?)), + other => Ok(Box::new(CompatProvider::from_env(other)?)), + } +} + +/// HTTP client shared by the adapters: rustls with bundled roots, a connect +/// timeout, and a read timeout long enough for streams that send keep-alives. +pub(crate) fn http_client() -> anyhow::Result { + Ok(reqwest::Client::builder() + .user_agent(concat!("gitagent/", env!("CARGO_PKG_VERSION"))) + .connect_timeout(std::time::Duration::from_secs(30)) + .read_timeout(std::time::Duration::from_secs(300)) + .build()?) +} + +/// A temperature as JSON with the digits as written (0.7, not 0.699999988). +pub(crate) fn temperature_json(t: f32) -> serde_json::Value { + serde_json::from_str(&t.to_string()).unwrap_or_else(|_| serde_json::json!(t)) +} diff --git a/crates/provider/src/openai.rs b/crates/provider/src/openai.rs new file mode 100644 index 00000000..0e13b6df --- /dev/null +++ b/crates/provider/src/openai.rs @@ -0,0 +1,412 @@ +//! OpenAI Responses API adapter: raw HTTP with streaming responses. +//! +//! Requests are stateless (`store: false`): the whole history is sent every +//! turn, and the model's output items (including encrypted reasoning) are +//! replayed unchanged from `Message::raw`. + +use std::collections::BTreeMap; +use std::time::Duration; + +use async_trait::async_trait; +use gitagent_loader::{Family, OpenAiCfg}; +use serde_json::{json, Value}; + +use crate::sse::SseDecoder; +use crate::{ + ChatRequest, ChatResponse, Content, Message, Provider, ProviderError, RawContent, Role, + StopReason, ToolCall, Usage, +}; + +pub const PROVIDER: &str = "openai"; +const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +/// 429 codes that mean billing or quota problems; retrying does not help. +const QUOTA_CODES: &[&str] = &[ + "insufficient_quota", + "credit_balance_exhausted", + "organization_spend_limit_exceeded", + "project_spend_limit_exceeded", + "organization_usage_limit_exceeded", +]; + +pub struct OpenAiProvider { + client: reqwest::Client, + base_url: String, + api_key: String, + cfg: OpenAiCfg, +} + +impl OpenAiProvider { + pub fn new(api_key: String, base_url: Option, cfg: OpenAiCfg) -> anyhow::Result { + let client = reqwest::Client::builder() + .user_agent(concat!("gitagent/", env!("CARGO_PKG_VERSION"))) + .connect_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(300)) + .build()?; + Ok(OpenAiProvider { + client, + base_url: base_url + .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) + .trim_end_matches('/') + .to_string(), + api_key, + cfg, + }) + } + + /// Reads `OPENAI_API_KEY` and, if set, `OPENAI_BASE_URL`. + pub fn from_env(cfg: OpenAiCfg) -> anyhow::Result { + let key = std::env::var("OPENAI_API_KEY") + .ok() + .filter(|k| !k.trim().is_empty()) + .ok_or_else(|| anyhow::anyhow!("OPENAI_API_KEY is not set"))?; + Self::new(key, std::env::var("OPENAI_BASE_URL").ok(), cfg) + } +} + +#[async_trait] +impl Provider for OpenAiProvider { + async fn complete(&self, req: &ChatRequest) -> Result { + let body = serde_json::to_vec(&request_body(req, &self.cfg)) + .map_err(|e| ProviderError::Fatal(e.to_string()))?; + let mut resp = self + .client + .post(format!("{}/responses", self.base_url)) + .bearer_auth(&self.api_key) + .header("content-type", "application/json") + .header("accept", "text/event-stream") + .body(body) + .send() + .await + .map_err(|e| ProviderError::Transient(format!("request failed: {e}")))?; + + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + return Err(classify_http_error(status.as_u16(), &text)); + } + let mut stream = StreamParser::default(); + while let Some(chunk) = resp + .chunk() + .await + .map_err(|e| ProviderError::Transient(format!("stream interrupted: {e}")))? + { + stream.feed(&chunk)?; + } + stream.finish() + } + + fn family(&self) -> Family { + Family::OpenAI + } +} + +/// Builds the request JSON (deterministic: serde_json sorts object keys). +pub(crate) fn request_body(req: &ChatRequest, cfg: &OpenAiCfg) -> Value { + let tools: Vec = req + .tools + .iter() + .map(|t| { + json!({ + "type": "function", + "name": t.name, + "description": t.description, + "parameters": t.input_schema, + "strict": false, + }) + }) + .collect(); + + let mut input = Vec::new(); + for m in &req.messages { + input.extend(message_items(m)); + } + + let mut reasoning = json!({"effort": req.effort.as_str()}); + if cfg.reasoning_summary != "none" { + reasoning["summary"] = json!(cfg.reasoning_summary); + } + let mut body = json!({ + "model": req.model, + "instructions": req.system, + "input": input, + "tools": tools, + "parallel_tool_calls": true, + "reasoning": reasoning, + "max_output_tokens": req.max_output_tokens, + "store": false, + "stream": true, + "include": ["reasoning.encrypted_content"], + }); + if let Some(t) = req.temperature { + body["temperature"] = crate::temperature_json(t); + } + if let Some(key) = &cfg.prompt_cache_key { + body["prompt_cache_key"] = json!(key); + } + body +} + +fn message_items(m: &Message) -> Vec { + if let Some(raw) = m.raw.as_ref().filter(|r| r.provider == PROVIDER) { + return raw.blocks.clone(); + } + let mut items = Vec::new(); + let mut text = Vec::new(); + for c in &m.content { + match c { + Content::Text { text: t } => text.push(t.as_str()), + Content::ToolUse { id, name, input } => items.push(json!({ + "type": "function_call", + "call_id": id, + "name": name, + "arguments": Value::Object(input.clone()).to_string(), + })), + Content::ToolResult { + tool_use_id, + content, + images, + .. + } => { + let output = if images.is_empty() { + json!(content) + } else { + let mut parts = vec![json!({"type": "input_text", "text": content})]; + parts.extend(images.iter().map(|i| { + json!({"type": "input_image", "image_url": format!("data:{};base64,{}", i.media_type, i.data)}) + })); + Value::Array(parts) + }; + items.push(json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output, + })) + } + } + } + if !text.is_empty() { + let (role, part) = match m.role { + Role::User => ("user", "input_text"), + Role::Assistant => ("assistant", "output_text"), + }; + let message = json!({ + "type": "message", + "role": role, + "content": [{"type": part, "text": text.join("\n")}], + }); + // An assistant's text precedes its calls; a user's notes follow the + // tool outputs they comment on. + match m.role { + Role::Assistant => items.insert(0, message), + Role::User => items.push(message), + } + } + items +} + +pub(crate) fn classify_http_error(status: u16, body: &str) -> ProviderError { + let parsed: Value = serde_json::from_str(body).unwrap_or(Value::Null); + let err = &parsed["error"]; + let kind = err["type"].as_str().unwrap_or(""); + let code = err["code"].as_str().unwrap_or(""); + let message = err["message"] + .as_str() + .map(str::to_string) + .unwrap_or_else(|| body.chars().take(500).collect()); + let detail = format!("HTTP {status} {kind} {code}: {message}"); + classify(status, kind, code, &message, detail) +} + +fn classify(status: u16, kind: &str, code: &str, message: &str, detail: String) -> ProviderError { + let lower = message.to_lowercase(); + if QUOTA_CODES.contains(&code) || kind == "insufficient_quota" { + ProviderError::Fatal(detail) + } else if code == "context_length_exceeded" + || lower.contains("context window") + || lower.contains("maximum context length") + { + ProviderError::ContextOverflow(detail) + } else if matches!(status, 408 | 409 | 429 | 500 | 502 | 503 | 504) + || matches!( + kind, + "server_error" | "rate_limit_error" | "service_unavailable_error" + ) + || matches!( + code, + "server_error" | "rate_limit_exceeded" | "server_is_overloaded" + ) + { + ProviderError::Transient(detail) + } else { + ProviderError::Fatal(detail) + } +} + +/// Assembles a streamed response. The final `response.completed` (or +/// `.incomplete`) event carries the whole response; finished output items +/// are also collected as they arrive, in case the final event omits them. +#[derive(Default)] +pub(crate) struct StreamParser { + sse: SseDecoder, + items: BTreeMap, + last: Option, +} + +impl StreamParser { + pub fn feed(&mut self, bytes: &[u8]) -> Result<(), ProviderError> { + for data in self.sse.feed(bytes) { + if data == "[DONE]" { + continue; + } + let ev: Value = serde_json::from_str(&data) + .map_err(|e| ProviderError::Transient(format!("bad stream event: {e}")))?; + match ev["type"].as_str().unwrap_or("") { + "response.output_item.done" => { + let index = ev["output_index"] + .as_u64() + .unwrap_or(self.items.len() as u64); + self.items.insert(index, ev["item"].clone()); + } + "response.completed" | "response.incomplete" => { + self.last = Some(ev["response"].clone()); + } + "response.failed" => { + let e = &ev["response"]["error"]; + let code = e["code"].as_str().unwrap_or(""); + let message = e["message"].as_str().unwrap_or(""); + return Err(classify( + 0, + "", + code, + message, + format!("response failed {code}: {message}"), + )); + } + "error" => { + // Shapes differ between versions: fields at the top level + // or under `error`. + let e = if ev["error"].is_object() { + &ev["error"] + } else { + &ev + }; + let kind = e["type"].as_str().unwrap_or(""); + let code = e["code"].as_str().unwrap_or(""); + let message = e["message"].as_str().unwrap_or(""); + return Err(classify( + 0, + kind, + code, + message, + format!("stream error {code}: {message}"), + )); + } + _ => {} + } + } + Ok(()) + } + + pub fn finish(self) -> Result { + let Some(resp) = self.last else { + return Err(ProviderError::Transient( + "stream ended before the response completed".into(), + )); + }; + let mut items: Vec = match resp["output"].as_array() { + Some(out) if !out.is_empty() => out.clone(), + _ => self.items.into_values().collect(), + }; + + let mut text = Vec::new(); + let mut reasoning = Vec::new(); + let mut tool_calls = Vec::new(); + let mut refused = false; + for item in &items { + match item["type"].as_str().unwrap_or("") { + "message" => { + for part in item["content"].as_array().into_iter().flatten() { + match part["type"].as_str().unwrap_or("") { + "output_text" => { + text.push(part["text"].as_str().unwrap_or("").to_string()) + } + "refusal" => { + refused = true; + text.push(part["refusal"].as_str().unwrap_or("").to_string()); + } + _ => {} + } + } + } + "reasoning" => { + for s in item["summary"].as_array().into_iter().flatten() { + if let Some(t) = s["text"].as_str().filter(|t| !t.is_empty()) { + reasoning.push(t.to_string()); + } + } + } + "function_call" => { + let args = item["arguments"].as_str().unwrap_or(""); + // Cut-off or invalid JSON goes back as a call with no arguments. + let arguments = serde_json::from_str::(args) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(); + tool_calls.push(ToolCall { + id: item["call_id"].as_str().unwrap_or("").to_string(), + name: item["name"].as_str().unwrap_or("").to_string(), + arguments, + }); + } + _ => {} + } + } + if !items + .iter() + .any(|i| matches!(i["type"].as_str(), Some("message" | "function_call"))) + { + items.push(json!({ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "(empty response)"}], + })); + } + + let usage = &resp["usage"]; + let count = |v: &Value| v.as_u64().unwrap_or(0); + let usage = Usage { + input_tokens: count(&usage["input_tokens"]), + output_tokens: count(&usage["output_tokens"]), + cache_read_tokens: count(&usage["input_tokens_details"]["cached_tokens"]), + cache_write_tokens: count(&usage["input_tokens_details"]["cache_write_tokens"]), + }; + + let incomplete = resp["incomplete_details"]["reason"].as_str(); + let stop_reason = match (resp["status"].as_str(), incomplete) { + (_, Some("max_output_tokens")) => StopReason::MaxTokens, + (_, Some("content_filter")) => StopReason::Refusal, + _ if refused && tool_calls.is_empty() => StopReason::Refusal, + _ if !tool_calls.is_empty() => StopReason::ToolUse, + (Some("completed"), _) => StopReason::EndTurn, + (status, reason) => { + StopReason::Other(reason.or(status).unwrap_or("unknown").to_string()) + } + }; + Ok(ChatResponse { + model: resp["model"].as_str().map(str::to_string), + text: text.join("\n"), + reasoning: (!reasoning.is_empty()).then(|| reasoning.join("\n\n")), + tool_calls, + usage, + stop_reason, + raw: Some(RawContent { + provider: PROVIDER.to_string(), + blocks: items, + }), + cleared_tool_results: 0, + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provider/src/openai/tests.rs b/crates/provider/src/openai/tests.rs new file mode 100644 index 00000000..59e4505e --- /dev/null +++ b/crates/provider/src/openai/tests.rs @@ -0,0 +1,352 @@ +use super::*; +use crate::{CachePlan, ToolSpec}; +use gitagent_loader::Effort; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +fn cfg() -> OpenAiCfg { + OpenAiCfg { + reasoning_summary: "auto".into(), + prompt_cache_key: Some("gitagent-v2".into()), + } +} + +fn request(messages: Vec) -> ChatRequest { + ChatRequest { + model: "gpt-test".into(), + system: "SYSTEM".into(), + messages, + tools: vec![ToolSpec { + name: "exec".into(), + description: "run".into(), + input_schema: json!({"type": "object"}), + }], + effort: Effort::High, + temperature: None, + max_output_tokens: 64000, + cache: CachePlan::default(), + server_masking: None, + } +} + +fn reasoning_item() -> Value { + json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "ENC"}) +} + +fn call_item() -> Value { + json!({"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "exec", "arguments": "{\"cmd\":\"ls\"}", "status": "completed"}) +} + +#[test] +fn request_body_replays_output_items_and_converts_the_rest() { + let assistant = Message { + role: Role::Assistant, + content: vec![], + raw: Some(RawContent { + provider: PROVIDER.into(), + blocks: vec![reasoning_item(), call_item()], + }), + }; + let results = Message::tool_results(vec![Content::ToolResult { + tool_use_id: "call_1".into(), + content: "a\nb".into(), + is_error: false, + images: vec![], + }]); + let body = request_body( + &request(vec![Message::user_text("task"), assistant, results]), + &cfg(), + ); + + assert_eq!(body["instructions"], "SYSTEM"); + assert_eq!(body["store"], false); + assert_eq!(body["stream"], true); + assert_eq!( + body["reasoning"], + json!({"effort": "high", "summary": "auto"}) + ); + assert_eq!(body["include"], json!(["reasoning.encrypted_content"])); + assert_eq!(body["prompt_cache_key"], "gitagent-v2"); + assert_eq!(body["max_output_tokens"], 64000); + assert!(body.get("temperature").is_none()); + assert_eq!( + body["tools"][0], + json!({"type": "function", "name": "exec", "description": "run", "parameters": {"type": "object"}, "strict": false}) + ); + let input = body["input"].as_array().unwrap(); + assert_eq!( + input[0], + json!({"type": "message", "role": "user", "content": [{"type": "input_text", "text": "task"}]}) + ); + assert_eq!(input[1], reasoning_item()); + assert_eq!(input[2], call_item()); + assert_eq!( + input[3], + json!({"type": "function_call_output", "call_id": "call_1", "output": "a\nb"}) + ); +} + +#[test] +fn converts_history_from_other_providers() { + let input = json!({"cmd": "ls"}).as_object().unwrap().clone(); + let assistant = Message { + role: Role::Assistant, + content: vec![ + Content::Text { + text: "Listing.".into(), + }, + Content::ToolUse { + id: "c9".into(), + name: "exec".into(), + input, + }, + ], + raw: Some(RawContent { + provider: "anthropic".into(), + blocks: vec![json!({"type": "thinking"})], + }), + }; + let items = message_items(&assistant); + assert_eq!(items[0]["content"][0]["type"], "output_text"); + assert_eq!(items[1]["type"], "function_call"); + assert_eq!(items[1]["arguments"], "{\"cmd\":\"ls\"}"); +} + +fn ev(v: Value) -> String { + format!("event: {}\ndata: {}\n\n", v["type"].as_str().unwrap(), v) +} + +fn stream() -> String { + let message = json!({"type": "message", "id": "msg_1", "role": "assistant", "content": [{"type": "output_text", "text": "Listing."}]}); + let reasoning = json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "Check files."}], "encrypted_content": "ENC"}); + let call = call_item(); + [ + ev(json!({"type": "response.created", "response": {"id": "resp_1", "status": "in_progress"}})), + ev(json!({"type": "response.output_item.done", "output_index": 0, "item": reasoning})), + ev(json!({"type": "response.output_text.delta", "delta": "List"})), + ev(json!({"type": "response.output_item.done", "output_index": 1, "item": message})), + ev(json!({"type": "response.function_call_arguments.delta", "delta": "{\"cmd\""})), + ev(json!({"type": "response.output_item.done", "output_index": 2, "item": call})), + ev(json!({"type": "response.completed", "response": { + "id": "resp_1", "model": "gpt-test-2026", "status": "completed", + "output": [reasoning, message, call], + "usage": {"input_tokens": 5000, "output_tokens": 120, + "input_tokens_details": {"cached_tokens": 4096}, + "output_tokens_details": {"reasoning_tokens": 80}} + }})), + ] + .concat() +} + +fn parse(s: &str, chunk: usize) -> Result { + let mut p = StreamParser::default(); + for c in s.as_bytes().chunks(chunk) { + p.feed(c)?; + } + p.finish() +} + +#[test] +fn assembles_a_streamed_response() { + for size in [1, 13, 4096] { + let r = parse(&stream(), size).unwrap(); + assert_eq!(r.model.as_deref(), Some("gpt-test-2026")); + assert_eq!(r.text, "Listing."); + assert_eq!(r.reasoning.as_deref(), Some("Check files.")); + assert_eq!(r.stop_reason, StopReason::ToolUse); + assert_eq!(r.tool_calls[0].id, "call_1"); + assert_eq!(r.tool_calls[0].arguments["cmd"], "ls"); + assert_eq!( + r.usage, + Usage { + input_tokens: 5000, + output_tokens: 120, + cache_read_tokens: 4096, + cache_write_tokens: 0 + } + ); + let raw = r.raw.unwrap(); + assert_eq!(raw.blocks.len(), 3); + assert_eq!(raw.blocks[0]["encrypted_content"], "ENC"); + } +} + +#[test] +fn falls_back_to_streamed_items_when_the_final_event_has_no_output() { + let call = call_item(); + let s = [ + ev(json!({"type": "response.output_item.done", "output_index": 0, "item": call})), + ev(json!({"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 10, "output_tokens": 2}}})), + ] + .concat(); + let r = parse(&s, 64).unwrap(); + assert_eq!(r.tool_calls.len(), 1); +} + +#[test] +fn incomplete_failed_and_truncated_streams() { + let s = ev( + json!({"type": "response.incomplete", "response": {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, "output": [], "usage": {}}}), + ); + let r = parse(&s, 64).unwrap(); + assert_eq!(r.stop_reason, StopReason::MaxTokens); + assert_eq!( + r.raw.unwrap().blocks[0]["content"][0]["text"], + "(empty response)" + ); + + let s = ev( + json!({"type": "response.failed", "response": {"status": "failed", "error": {"code": "server_error", "message": "boom"}}}), + ); + assert!(matches!(parse(&s, 64), Err(ProviderError::Transient(_)))); + + let s = ev(json!({"type": "error", "code": "context_length_exceeded", "message": "too long"})); + assert!(matches!( + parse(&s, 64), + Err(ProviderError::ContextOverflow(_)) + )); + + let cut = &stream()[..stream().find("event: response.completed").unwrap()]; + assert!(matches!(parse(cut, 64), Err(ProviderError::Transient(_)))); +} + +#[test] +fn refusals_are_reported() { + let message = json!({"type": "message", "role": "assistant", "content": [{"type": "refusal", "refusal": "I can't help with that."}]}); + let s = ev( + json!({"type": "response.completed", "response": {"status": "completed", "output": [message], "usage": {}}}), + ); + let r = parse(&s, 64).unwrap(); + assert_eq!(r.stop_reason, StopReason::Refusal); +} + +#[test] +fn classifies_http_errors() { + let body = |t: &str, c: &str, m: &str| { + json!({"error": {"type": t, "code": c, "message": m}}).to_string() + }; + assert!(matches!( + classify_http_error( + 429, + &body("rate_limit_error", "rate_limit_exceeded", "slow down") + ), + ProviderError::Transient(_) + )); + assert!(matches!( + classify_http_error( + 429, + &body( + "insufficient_quota", + "credit_balance_exhausted", + "no credits" + ) + ), + ProviderError::Fatal(_) + )); + assert!(matches!( + classify_http_error( + 400, + &body( + "invalid_request_error", + "context_length_exceeded", + "too long" + ) + ), + ProviderError::ContextOverflow(_) + )); + assert!(matches!( + classify_http_error( + 401, + &body("invalid_request_error", "invalid_api_key", "bad key") + ), + ProviderError::Fatal(_) + )); + assert!(matches!( + classify_http_error(503, "upstream unavailable"), + ProviderError::Transient(_) + )); +} + +#[tokio::test] +async fn completes_against_a_local_server() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/v1", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut req = Vec::new(); + let mut buf = [0u8; 8192]; + loop { + let n = sock.read(&mut buf).await.unwrap(); + req.extend_from_slice(&buf[..n]); + let text = String::from_utf8_lossy(&req).to_string(); + if let Some(h) = text.find("\r\n\r\n") { + let len: usize = text[..h] + .lines() + .find_map(|l| { + l.to_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().parse().unwrap()) + }) + .unwrap_or(0); + if req.len() >= h + 4 + len { + break; + } + } + } + let resp = format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n{}", + stream() + ); + sock.write_all(resp.as_bytes()).await.unwrap(); + sock.shutdown().await.unwrap(); + String::from_utf8_lossy(&req).into_owned() + }); + let provider = OpenAiProvider::new("sk-test".into(), Some(url), cfg()).unwrap(); + let r = provider + .complete(&request(vec![Message::user_text("task")])) + .await + .unwrap(); + assert_eq!(r.tool_calls[0].name, "exec"); + let raw = server.await.unwrap(); + assert!(raw.starts_with("POST /v1/responses HTTP/1.1")); + assert!(raw.to_lowercase().contains("authorization: bearer sk-test")); +} + +#[test] +fn tool_results_with_images_become_input_parts() { + let msg = Message::tool_results(vec![Content::ToolResult { + tool_use_id: "c".into(), + content: "chart".into(), + is_error: false, + images: vec![crate::ImageData { + media_type: "image/jpeg".into(), + data: "BBBB".into(), + }], + }]); + let items = message_items(&msg); + assert_eq!( + items[0]["output"], + json!([ + {"type": "input_text", "text": "chart"}, + {"type": "input_image", "image_url": "data:image/jpeg;base64,BBBB"} + ]) + ); +} + +#[test] +fn notes_in_a_user_turn_follow_the_tool_outputs() { + let msg = Message::tool_results(vec![ + Content::ToolResult { + tool_use_id: "call_1".into(), + content: "ok".into(), + is_error: false, + images: vec![], + }, + Content::Text { + text: "Time check".into(), + }, + ]); + let items = message_items(&msg); + assert_eq!(items[0]["type"], "function_call_output"); + assert_eq!(items[1]["role"], "user"); + assert_eq!(items[1]["content"][0]["text"], "Time check"); +} diff --git a/crates/provider/src/scripted.rs b/crates/provider/src/scripted.rs new file mode 100644 index 00000000..1d2d6449 --- /dev/null +++ b/crates/provider/src/scripted.rs @@ -0,0 +1,176 @@ +//! A provider that replays responses from a YAML script. Used by tests and by +//! `--model scripted:` for dry runs without network access. + +use std::path::Path; +use std::sync::Mutex; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use gitagent_loader::Family; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::{ChatRequest, ChatResponse, Provider, ProviderError, StopReason, ToolCall, Usage}; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct ScriptedTurn { + #[serde(default)] + text: String, + #[serde(default)] + tool_calls: Vec, + /// Simulates a safety-classifier refusal. + #[serde(default)] + refusal: bool, + /// Simulates a provider error: `transient`, `context_overflow` or `fatal`. + #[serde(default)] + error: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct ScriptedCall { + name: String, + #[serde(default)] + arguments: Map, +} + +pub struct ScriptedProvider { + turns: Vec, + state: Mutex, +} + +#[derive(Default)] +struct State { + next: usize, + requests: Vec, +} + +impl ScriptedProvider { + /// Parses a script: a YAML list of turns, each with optional `text`, + /// `tool_calls: [{name, arguments}]`, `refusal: true` or `error: `. + pub fn from_yaml(yaml: &str) -> Result { + let turns: Vec = + serde_saphyr::from_str(yaml).context("parsing scripted provider fixture")?; + Ok(ScriptedProvider { + turns, + state: Mutex::new(State::default()), + }) + } + + pub fn from_file(path: &Path) -> Result { + let yaml = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + Self::from_yaml(&yaml) + } + + /// Every request received so far, in order. + pub fn requests(&self) -> Vec { + self.state.lock().unwrap().requests.clone() + } +} + +#[async_trait] +impl Provider for ScriptedProvider { + async fn complete(&self, req: &ChatRequest) -> Result { + let mut state = self.state.lock().unwrap(); + state.requests.push(req.clone()); + let index = state.next; + let turn = self + .turns + .get(index) + .ok_or_else(|| ProviderError::Fatal(format!("script ended after {index} turns")))?; + state.next += 1; + if let Some(kind) = &turn.error { + return Err(match kind.as_str() { + "transient" => ProviderError::Transient("scripted".into()), + "context_overflow" => ProviderError::ContextOverflow("scripted".into()), + _ => ProviderError::Fatal("scripted".into()), + }); + } + + let tool_calls: Vec = turn + .tool_calls + .iter() + .enumerate() + .map(|(i, c)| ToolCall { + id: format!("call_{}_{}", index + 1, i + 1), + name: c.name.clone(), + arguments: c.arguments.clone(), + }) + .collect(); + let prompt_bytes: usize = req.system.len() + + req + .messages + .iter() + .map(|m| serde_json::to_string(m).map_or(0, |s| s.len())) + .sum::(); + Ok(ChatResponse { + model: None, + text: turn.text.clone(), + reasoning: None, + stop_reason: if turn.refusal { + StopReason::Refusal + } else if tool_calls.is_empty() { + StopReason::EndTurn + } else { + StopReason::ToolUse + }, + tool_calls, + usage: Usage { + input_tokens: (prompt_bytes / 4) as u64, + output_tokens: (turn.text.len() / 4) as u64 + 10, + cache_read_tokens: 0, + cache_write_tokens: 0, + }, + raw: None, + cleared_tool_results: 0, + }) + } + + fn family(&self) -> Family { + Family::Compat + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CachePlan, Message}; + use gitagent_loader::Effort; + + fn request() -> ChatRequest { + ChatRequest { + model: "m".into(), + system: "sys".into(), + messages: vec![Message::user_text("task")], + tools: vec![], + effort: Effort::High, + temperature: None, + max_output_tokens: 100, + cache: CachePlan::default(), + server_masking: None, + } + } + + #[tokio::test] + async fn replays_turns_then_errors() { + let p = ScriptedProvider::from_yaml( + "- text: looking\n tool_calls:\n - name: exec\n arguments: {cmd: echo hi, yield_ms: 500}\n- tool_calls:\n - name: task_complete\n", + ) + .unwrap(); + let r1 = p.complete(&request()).await.unwrap(); + assert_eq!(r1.text, "looking"); + assert_eq!(r1.tool_calls[0].name, "exec"); + assert_eq!(r1.tool_calls[0].arguments["cmd"], "echo hi"); + assert_eq!(r1.tool_calls[0].arguments["yield_ms"], 500); + assert_eq!(r1.stop_reason, StopReason::ToolUse); + let r2 = p.complete(&request()).await.unwrap(); + assert_eq!(r2.tool_calls[0].name, "task_complete"); + assert!(matches!( + p.complete(&request()).await, + Err(ProviderError::Fatal(_)) + )); + assert_eq!(p.requests().len(), 3); + } +} diff --git a/crates/provider/src/sse.rs b/crates/provider/src/sse.rs new file mode 100644 index 00000000..567c4304 --- /dev/null +++ b/crates/provider/src/sse.rs @@ -0,0 +1,44 @@ +//! Server-sent events: splits a byte stream into the `data` payload of each event. + +#[derive(Default)] +pub(crate) struct SseDecoder { + buf: Vec, +} + +impl SseDecoder { + /// Adds bytes and returns the data of every event completed by them. + pub fn feed(&mut self, bytes: &[u8]) -> Vec { + // JSON strings escape CR, so dropping it only normalizes line endings. + self.buf.extend(bytes.iter().filter(|&&b| b != b'\r')); + let mut events = Vec::new(); + while let Some(end) = self.buf.windows(2).position(|w| w == b"\n\n") { + let event: Vec = self.buf.drain(..end + 2).collect(); + let text = String::from_utf8_lossy(&event); + let data: Vec<&str> = text + .lines() + .filter_map(|l| l.strip_prefix("data:")) + .map(|d| d.strip_prefix(' ').unwrap_or(d)) + .collect(); + if !data.is_empty() { + events.push(data.join("\n")); + } + } + events + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_events_across_chunks_and_line_endings() { + let mut d = SseDecoder::default(); + assert!(d.feed(b"event: a\r\ndata: {\"x\":").is_empty()); + assert_eq!( + d.feed(b"1}\r\n\r\n: comment\n\ndata: two\n\n"), + ["{\"x\":1}", "two"] + ); + assert_eq!(d.feed(b"data: a\ndata: b\n\n"), ["a\nb"]); + } +} diff --git a/crates/provider/src/test_server.rs b/crates/provider/src/test_server.rs new file mode 100644 index 00000000..25e99dab --- /dev/null +++ b/crates/provider/src/test_server.rs @@ -0,0 +1,52 @@ +//! A one-request HTTP server for adapter tests. + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +/// Serves one response and returns the server's base URL and a handle that +/// yields the raw request it received. +pub(crate) async fn serve_once( + status: &'static str, + content_type: &'static str, + body: String, +) -> (String, tokio::task::JoinHandle) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let handle = tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut req = Vec::new(); + let mut buf = [0u8; 8192]; + loop { + let n = sock.read(&mut buf).await.unwrap(); + req.extend_from_slice(&buf[..n]); + let text = String::from_utf8_lossy(&req).to_string(); + if let Some(h) = text.find("\r\n\r\n") { + let len: usize = text[..h] + .lines() + .find_map(|l| { + l.to_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().parse().unwrap()) + }) + .unwrap_or(0); + if req.len() >= h + 4 + len { + break; + } + } + if n == 0 { + break; + } + } + let resp = format!( + "HTTP/1.1 {status}\r\ncontent-type: {content_type}\r\nconnection: close\r\n\r\n{body}" + ); + sock.write_all(resp.as_bytes()).await.unwrap(); + sock.shutdown().await.unwrap(); + String::from_utf8_lossy(&req).into_owned() + }); + (url, handle) +} + +/// The JSON body of a raw HTTP request. +pub(crate) fn body_of(raw: &str) -> serde_json::Value { + serde_json::from_str(&raw[raw.find("\r\n\r\n").unwrap() + 4..]).unwrap() +} diff --git a/crates/provider/src/types.rs b/crates/provider/src/types.rs new file mode 100644 index 00000000..b2b94fc3 --- /dev/null +++ b/crates/provider/src/types.rs @@ -0,0 +1,200 @@ +use gitagent_loader::Effort; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Role { + User, + Assistant, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Content { + Text { + text: String, + }, + ToolUse { + id: String, + name: String, + input: Map, + }, + ToolResult { + tool_use_id: String, + content: String, + is_error: bool, + /// Images returned by the tool (e.g. `image_read`). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + images: Vec, + }, +} + +/// A base64-encoded image. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImageData { + /// `image/png`, `image/jpeg`, `image/gif` or `image/webp`. + pub media_type: String, + pub data: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Message { + pub role: Role, + pub content: Vec, + /// The provider's own content blocks for this message, replayed verbatim to + /// the same provider (thinking blocks and their signatures must come back + /// unchanged). Other providers use `content`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RawContent { + pub provider: String, + pub blocks: Vec, +} + +impl Message { + pub fn user_text(text: impl Into) -> Self { + Message { + role: Role::User, + content: vec![Content::Text { text: text.into() }], + raw: None, + } + } + + pub fn tool_results(results: Vec) -> Self { + Message { + role: Role::User, + content: results, + raw: None, + } + } +} + +/// A tool as offered to the model. `input_schema` is a JSON Schema object. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ToolSpec { + pub name: String, + pub description: String, + pub input_schema: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ToolCall { + pub id: String, + pub name: String, + pub arguments: Map, +} + +/// Where to place prompt-cache breakpoints (providers without explicit +/// breakpoints ignore this). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CachePlan { + pub system: bool, + /// Number of trailing messages that get a breakpoint. + pub last_messages: u8, + /// Ask for the one-hour cache lifetime instead of the five-minute one. + pub one_hour: bool, +} + +impl CachePlan { + /// Builds a plan from a profile's `cache.breakpoints` list: `system`, + /// `last_message`, `last_two_messages`. Unknown names are ignored. + pub fn from_breakpoints(names: &[String]) -> Self { + let mut plan = CachePlan::default(); + for name in names { + match name.as_str() { + "system" => plan.system = true, + "last_message" => plan.last_messages = plan.last_messages.max(1), + "last_two_messages" => plan.last_messages = plan.last_messages.max(2), + _ => {} + } + } + plan + } +} + +/// Provider-side clearing of old tool results (Anthropic context editing). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServerMasking { + /// Start clearing once the prompt reaches this many tokens. + pub trigger_tokens: u32, + /// Most recent tool results to keep. + pub keep: u32, + /// Clear only when at least this much can go (keeps clears batched). + pub clear_at_least: u32, +} + +#[derive(Debug, Clone)] +pub struct ChatRequest { + pub model: String, + pub system: String, + pub messages: Vec, + pub tools: Vec, + pub effort: Effort, + pub temperature: Option, + pub max_output_tokens: u32, + pub cache: CachePlan, + pub server_masking: Option, +} + +/// Token usage for one call. `input_tokens` is the full prompt, including +/// tokens read from or written to the cache. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Usage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StopReason { + EndTurn, + ToolUse, + MaxTokens, + /// The provider's safety classifiers declined the request. + Refusal, + Other(String), +} + +#[derive(Debug, Clone)] +pub struct ChatResponse { + /// The model that produced the response, when the provider reports it + /// (it can differ from the requested one after a refusal fallback). + pub model: Option, + pub text: String, + pub reasoning: Option, + pub tool_calls: Vec, + pub usage: Usage, + pub stop_reason: StopReason, + /// Provider content blocks to store with the assistant message. + pub raw: Option, + /// Tool results the provider cleared from its copy of the history. + pub cleared_tool_results: u64, +} + +impl ChatResponse { + /// The assistant message to append to the history. + pub fn to_message(&self) -> Message { + let mut content = Vec::new(); + if !self.text.is_empty() { + content.push(Content::Text { + text: self.text.clone(), + }); + } + content.extend(self.tool_calls.iter().map(|c| Content::ToolUse { + id: c.id.clone(), + name: c.name.clone(), + input: c.arguments.clone(), + })); + Message { + role: Role::Assistant, + content, + raw: self.raw.clone(), + } + } +} diff --git a/crates/record/Cargo.toml b/crates/record/Cargo.toml new file mode 100644 index 00000000..00ac828e --- /dev/null +++ b/crates/record/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "gitagent-record" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/record/src/atif.rs b/crates/record/src/atif.rs new file mode 100644 index 00000000..58add16f --- /dev/null +++ b/crates/record/src/atif.rs @@ -0,0 +1,352 @@ +//! Agent Trajectory Interchange Format (ATIF) as validated by Harbor. +//! +//! Field names and shapes follow Harbor's `harbor.models.trajectories` models +//! (schema `ATIF-v1.8`). Harbor rejects unknown keys, so anything of our own +//! goes into an `extra` object, never a new field. + +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +pub const SCHEMA_VERSION: &str = "ATIF-v1.8"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Trajectory { + pub schema_version: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + pub agent: Agent, + pub steps: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub notes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub final_metrics: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Agent { + pub name: String, + pub version: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_definitions: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra: Option>, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Source { + System, + User, + Agent, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Step { + pub step_id: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timestamp: Option, + pub source: Source, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metrics: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ToolCall { + pub tool_call_id: String, + pub function_name: String, + /// Must be a JSON object. + pub arguments: Map, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Observation { + pub results: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ObservationResult { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_call_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + /// Trajectories of sub-agents this call started. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_trajectory_ref: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra: Option>, +} + +/// Harbor requires `trajectory_id` or `trajectory_path`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SubagentTrajectoryRef { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trajectory_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trajectory_path: Option, +} + +/// Per-step token usage. `prompt_tokens` includes cached tokens; +/// `cached_tokens` is the cached subset. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct Metrics { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cached_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cost_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra: Option>, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct FinalMetrics { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_prompt_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_completion_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_cached_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_cost_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_steps: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra: Option>, +} + +impl Step { + fn new(source: Source, message: impl Into) -> Self { + Step { + step_id: 0, + timestamp: Some(crate::utc_now_rfc3339()), + source, + model_name: None, + reasoning_effort: None, + message: message.into(), + reasoning_content: None, + tool_calls: None, + observation: None, + metrics: None, + extra: None, + } + } + + pub fn system(message: impl Into) -> Self { + Self::new(Source::System, message) + } + + pub fn user(message: impl Into) -> Self { + Self::new(Source::User, message) + } + + pub fn agent(message: impl Into) -> Self { + Self::new(Source::Agent, message) + } +} + +/// Holds the trajectory in memory and rewrites it to disk on every `flush`. +pub struct TrajectoryWriter { + path: PathBuf, + trajectory: Trajectory, +} + +impl TrajectoryWriter { + pub fn new(path: impl Into, session_id: Option, agent: Agent) -> Self { + TrajectoryWriter { + path: path.into(), + trajectory: Trajectory { + schema_version: SCHEMA_VERSION.to_string(), + session_id, + agent, + steps: Vec::new(), + notes: None, + final_metrics: None, + extra: None, + }, + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn trajectory(&self) -> &Trajectory { + &self.trajectory + } + + /// Appends a step, assigning the next sequential `step_id`. + pub fn push(&mut self, mut step: Step) -> u32 { + step.step_id = self.trajectory.steps.len() as u32 + 1; + let id = step.step_id; + self.trajectory.steps.push(step); + id + } + + pub fn set_final_metrics(&mut self, metrics: FinalMetrics) { + self.trajectory.final_metrics = Some(metrics); + } + + pub fn set_extra(&mut self, key: &str, value: Value) { + self.trajectory + .extra + .get_or_insert_with(Map::new) + .insert(key.to_string(), value); + } + + /// Validates and writes the whole trajectory. Harbor needs at least one step, + /// so an empty trajectory is not written. + pub fn flush(&self) -> Result<()> { + if self.trajectory.steps.is_empty() { + return Ok(()); + } + validate(&self.trajectory)?; + let json = serde_json::to_vec_pretty(&self.trajectory)?; + crate::write_atomic(&self.path, &json) + } +} + +/// Checks the structural rules Harbor enforces when it loads a trajectory. +pub fn validate(t: &Trajectory) -> Result<()> { + if t.steps.is_empty() { + bail!("trajectory has no steps"); + } + for (i, step) in t.steps.iter().enumerate() { + if step.step_id as usize != i + 1 { + bail!("step {} has step_id {}", i + 1, step.step_id); + } + let agent_only = step.model_name.is_some() + || step.reasoning_effort.is_some() + || step.reasoning_content.is_some() + || step.tool_calls.is_some() + || step.metrics.is_some(); + if agent_only && step.source != Source::Agent { + bail!( + "step {} has agent-only fields on a non-agent step", + step.step_id + ); + } + if let Some(obs) = &step.observation { + let calls = step.tool_calls.as_deref().unwrap_or_default(); + for r in &obs.results { + if let Some(id) = &r.source_call_id { + if !calls.iter().any(|c| &c.tool_call_id == id) { + bail!( + "step {} observation refers to unknown tool call {id}", + step.step_id + ); + } + } + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn agent() -> Agent { + Agent { + name: "gitagent".into(), + version: "0.1.0".into(), + model_name: Some("test/model".into()), + tool_definitions: None, + extra: None, + } + } + + #[test] + fn writes_sequential_steps_with_matching_observations() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("trajectory.json"); + let mut w = TrajectoryWriter::new(&path, Some("s1".into()), agent()); + w.push(Step::user("do the task")); + let mut step = Step::agent("running it"); + let mut args = Map::new(); + args.insert("cmd".into(), json!("echo hi")); + step.tool_calls = Some(vec![ToolCall { + tool_call_id: "c1".into(), + function_name: "exec".into(), + arguments: args, + }]); + step.observation = Some(Observation { + results: vec![ObservationResult { + source_call_id: Some("c1".into()), + content: Some("hi".into()), + subagent_trajectory_ref: None, + extra: None, + }], + }); + step.metrics = Some(Metrics { + prompt_tokens: Some(100), + completion_tokens: Some(10), + cached_tokens: Some(80), + ..Default::default() + }); + assert_eq!(w.push(step), 2); + w.flush().unwrap(); + + let back: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(back["schema_version"], "ATIF-v1.8"); + assert_eq!(back["steps"][0]["source"], "user"); + assert_eq!(back["steps"][1]["step_id"], 2); + assert_eq!( + back["steps"][1]["tool_calls"][0]["arguments"]["cmd"], + "echo hi" + ); + // Unset optional fields are omitted, not written as null. + assert!(back["steps"][0].get("metrics").is_none()); + } + + #[test] + fn rejects_observation_for_unknown_call() { + let mut w = TrajectoryWriter::new("/nonexistent/t.json", None, agent()); + let mut step = Step::agent(""); + step.observation = Some(Observation { + results: vec![ObservationResult { + source_call_id: Some("missing".into()), + content: None, + subagent_trajectory_ref: None, + extra: None, + }], + }); + w.push(step); + assert!(w.flush().is_err()); + } + + #[test] + fn rejects_agent_fields_on_user_step() { + let mut w = TrajectoryWriter::new("/nonexistent/t.json", None, agent()); + let mut step = Step::user("hi"); + step.metrics = Some(Metrics::default()); + w.push(step); + assert!(w.flush().is_err()); + } +} diff --git a/crates/record/src/counters.rs b/crates/record/src/counters.rs new file mode 100644 index 00000000..87fc5b06 --- /dev/null +++ b/crates/record/src/counters.rs @@ -0,0 +1,92 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use anyhow::Result; +use serde::Serialize; + +/// Fired/consumed counters for every feature (BLUEPRINT §8). +/// +/// Plain counters serialize as numbers, labelled ones (such as `tool_calls`) +/// as an object keyed by label. +#[derive(Debug, Default, Clone, Serialize, PartialEq, Eq)] +pub struct Counters { + #[serde(flatten)] + plain: BTreeMap, + #[serde(flatten)] + labelled: BTreeMap>, +} + +impl Counters { + pub fn new() -> Self { + Self::default() + } + + pub fn inc(&mut self, name: &str) { + self.add(name, 1); + } + + pub fn add(&mut self, name: &str, n: u64) { + *self.plain.entry(name.to_string()).or_default() += n; + } + + pub fn inc_labelled(&mut self, name: &str, label: &str) { + *self + .labelled + .entry(name.to_string()) + .or_default() + .entry(label.to_string()) + .or_default() += 1; + } + + pub fn get(&self, name: &str) -> u64 { + self.plain.get(name).copied().unwrap_or(0) + } + + pub fn get_labelled(&self, name: &str, label: &str) -> u64 { + self.labelled + .get(name) + .and_then(|m| m.get(label)) + .copied() + .unwrap_or(0) + } + + pub fn to_json(&self) -> serde_json::Value { + serde_json::to_value(self).unwrap_or_default() + } + + pub fn write(&self, path: &Path) -> Result<()> { + crate::write_atomic(path, &serde_json::to_vec_pretty(self)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_and_serializes() { + let mut c = Counters::new(); + c.inc("turns"); + c.inc("turns"); + c.add("input_tokens", 120); + c.inc_labelled("tool_calls", "exec"); + c.inc_labelled("tool_calls", "exec"); + c.inc_labelled("tool_calls", "read_file"); + assert_eq!(c.get("turns"), 2); + assert_eq!(c.get("missing"), 0); + assert_eq!(c.get_labelled("tool_calls", "exec"), 2); + assert_eq!( + c.to_json(), + serde_json::json!({ + "input_tokens": 120, + "tool_calls": {"exec": 2, "read_file": 1}, + "turns": 2 + }) + ); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested/counters.json"); + c.write(&path).unwrap(); + assert!(path.exists()); + } +} diff --git a/crates/record/src/events.rs b/crates/record/src/events.rs new file mode 100644 index 00000000..56261aa6 --- /dev/null +++ b/crates/record/src/events.rs @@ -0,0 +1,116 @@ +//! `--events jsonl`: one JSON object per line, for logs and tools that drive +//! gitagent. Nothing secret is ever put in an event. + +use std::io::Write; +use std::sync::{Arc, Mutex}; + +use serde_json::{Map, Value}; + +/// Writes events as JSON lines, or hands each one to a callback. Disabled +/// sinks drop everything. +#[derive(Clone, Default)] +pub struct Events { + out: Option>>>, + callback: Option>, +} + +impl Events { + pub fn disabled() -> Self { + Events::default() + } + + /// Every event goes to `f` as the same object the JSON lines carry. + pub fn to_callback(f: impl Fn(Value) + Send + Sync + 'static) -> Self { + Events { + out: None, + callback: Some(Arc::new(f)), + } + } + + pub fn stdout() -> Self { + Self::to_writer(Box::new(std::io::stdout())) + } + + pub fn to_writer(w: Box) -> Self { + Events { + out: Some(Arc::new(Mutex::new(w))), + callback: None, + } + } + + pub fn enabled(&self) -> bool { + self.out.is_some() || self.callback.is_some() + } + + /// Emits `{"event": kind, "ts": ..., ...fields}`. Write errors are ignored: + /// a closed pipe must not stop the run. + pub fn emit(&self, kind: &str, fields: Value) { + if self.out.is_none() && self.callback.is_none() { + return; + } + let mut obj = Map::new(); + obj.insert("event".into(), Value::String(kind.into())); + obj.insert("ts".into(), Value::String(crate::utc_now_rfc3339())); + if let Value::Object(f) = fields { + obj.extend(f); + } + if let Some(f) = &self.callback { + f(Value::Object(obj.clone())); + } + let Some(out) = &self.out else { + return; + }; + let mut line = serde_json::to_string(&Value::Object(obj)).unwrap_or_default(); + line.push('\n'); + if let Ok(mut w) = out.lock() { + let _ = w.write_all(line.as_bytes()); + let _ = w.flush(); + } + } +} + +/// A shared in-memory buffer, for tests that read events back. +#[derive(Clone, Default)] +pub struct SharedBuffer(pub Arc>>); + +impl Write for SharedBuffer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl SharedBuffer { + pub fn lines(&self) -> Vec { + String::from_utf8_lossy(&self.0.lock().unwrap()) + .lines() + .filter_map(|l| serde_json::from_str(l).ok()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn writes_one_object_per_line() { + let buf = SharedBuffer::default(); + let events = Events::to_writer(Box::new(buf.clone())); + events.emit("turn_start", json!({"turn": 1})); + events.emit("run_end", json!({"outcome": "completed"})); + let lines = buf.lines(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0]["event"], "turn_start"); + assert_eq!(lines[0]["turn"], 1); + assert!(lines[0]["ts"].as_str().unwrap().ends_with('Z')); + assert_eq!(lines[1]["outcome"], "completed"); + // A disabled sink does nothing. + Events::disabled().emit("x", json!({})); + } +} diff --git a/crates/record/src/lib.rs b/crates/record/src/lib.rs new file mode 100644 index 00000000..1ef875dd --- /dev/null +++ b/crates/record/src/lib.rs @@ -0,0 +1,30 @@ +//! Run records: the ATIF trajectory, feature counters and timestamps. +//! +//! Both files are rewritten atomically after every turn so that a run killed +//! by a timeout still leaves a complete record of everything up to that turn. + +pub mod atif; +mod counters; +mod events; +mod time; + +pub use counters::Counters; +pub use events::{Events, SharedBuffer}; +pub use time::utc_now_rfc3339; + +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Writes `bytes` to `path` via a temporary sibling file and a rename, so readers +/// never see a half-written file. +pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?; + } + let mut tmp = path.as_os_str().to_owned(); + tmp.push(".tmp"); + std::fs::write(&tmp, bytes).with_context(|| format!("writing {}", path.display()))?; + std::fs::rename(&tmp, path).with_context(|| format!("renaming into {}", path.display()))?; + Ok(()) +} diff --git a/crates/record/src/time.rs b/crates/record/src/time.rs new file mode 100644 index 00000000..5765a40a --- /dev/null +++ b/crates/record/src/time.rs @@ -0,0 +1,56 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Current UTC time as `YYYY-MM-DDTHH:MM:SS.mmmZ`. +pub fn utc_now_rfc3339() -> String { + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + format_unix_millis(since_epoch.as_millis() as i64) +} + +fn format_unix_millis(millis: i64) -> String { + let secs = millis.div_euclid(1000); + let ms = millis.rem_euclid(1000); + let days = secs.div_euclid(86_400); + let sod = secs.rem_euclid(86_400); + let (y, m, d) = civil_from_days(days); + format!( + "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{ms:03}Z", + sod / 3600, + (sod % 3600) / 60, + sod % 60 + ) +} + +/// Days since 1970-01-01 to (year, month, day) in the proleptic Gregorian calendar. +/// Howard Hinnant's `civil_from_days` algorithm. +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + let y = yoe + era * 400 + i64::from(m <= 2); + (y, m, d) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn formats_known_instants() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(1_791_986_096_789), + "2026-10-14T13:54:56.789Z" + ); + } +} diff --git a/crates/repo/Cargo.toml b/crates/repo/Cargo.toml new file mode 100644 index 00000000..0ba292a4 --- /dev/null +++ b/crates/repo/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "gitagent-repo" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/repo/src/lib.rs b/crates/repo/src/lib.rs new file mode 100644 index 00000000..69068fdb --- /dev/null +++ b/crates/repo/src/lib.rs @@ -0,0 +1,290 @@ +//! Repo mode (BLUEPRINT §5): clone a repository, work on a new branch, +//! commit when the task is done, and push only when asked. +//! +//! Rules: never commit to or push the base branch, never force-push, never +//! rewrite history. A token (from `GITAGENT_GIT_TOKEN` or `GITHUB_TOKEN`) +//! reaches git only through a `GIT_ASKPASS` helper: it is never put in a URL, +//! in `.git/config`, in a credential store, or in any log. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use anyhow::{bail, Context, Result}; + +/// Environment variable the askpass helper reads the token from. It is set +/// only on git commands that talk to the remote. +const TOKEN_VAR: &str = "GITAGENT_ASKPASS_TOKEN"; + +#[derive(Debug, Clone)] +pub struct RepoOptions { + pub url: String, + /// Branch to start from; default: the remote's default branch. + pub base: Option, + /// Continue an existing branch instead of creating one. + pub branch: Option, + /// Where to clone; must not exist yet (or be empty). + pub dir: PathBuf, + pub branch_prefix: String, + /// `Name ` for commits. + pub author: String, + pub token: Option, + /// A private directory for the askpass helper. + pub helper_dir: PathBuf, +} + +#[derive(Debug)] +pub struct Repo { + pub dir: PathBuf, + pub branch: String, + pub base: String, + /// True when an existing branch was checked out. + pub resumed: bool, + author_name: String, + author_email: String, + askpass: Option, + token: Option, +} + +impl Repo { + /// Clones the repository and checks out the working branch. + pub fn clone_for_task(opts: &RepoOptions, task: &str) -> Result { + if opts.dir.exists() + && std::fs::read_dir(&opts.dir).map_or(true, |mut d| d.next().is_some()) + { + bail!("{} already exists and is not empty", opts.dir.display()); + } + let askpass = match &opts.token { + Some(_) => Some(write_askpass(&opts.helper_dir)?), + None => None, + }; + let (name, email) = split_author(&opts.author); + let mut repo = Repo { + dir: opts.dir.clone(), + branch: String::new(), + base: String::new(), + resumed: false, + author_name: name, + author_email: email, + askpass, + token: opts.token.clone(), + }; + + let parent = opts.dir.parent().unwrap_or(Path::new(".")); + std::fs::create_dir_all(parent)?; + repo.remote_git( + parent, + &[ + "clone", + "--filter=blob:none", + "--no-tags", + &opts.url, + &opts.dir.to_string_lossy(), + ], + ) + .context("cloning the repository")?; + + repo.base = match &opts.base { + Some(b) => b.clone(), + None => repo.default_branch()?, + }; + let remote_branches = repo.remote_branches()?; + if !remote_branches.contains(&repo.base) { + bail!( + "the base branch {:?} does not exist on the remote", + repo.base + ); + } + + match &opts.branch { + Some(b) if *b == repo.base => { + bail!("refusing to work on the base branch {b:?}; pick another --branch") + } + Some(b) if remote_branches.contains(b) => { + repo.git(&["checkout", "-B", b, &format!("origin/{b}")])?; + repo.branch = b.clone(); + repo.resumed = true; + } + Some(b) => { + repo.git(&["checkout", "-b", b, &format!("origin/{}", repo.base)])?; + repo.branch = b.clone(); + } + None => { + let name = format!("{}{}-{}", opts.branch_prefix, slug(task), short_id()); + repo.git(&["checkout", "-b", &name, &format!("origin/{}", repo.base)])?; + repo.branch = name; + } + } + if repo.branch == repo.base { + bail!("refusing to work on the base branch {:?}", repo.base); + } + Ok(repo) + } + + /// Commits every change. Returns the commit id, or `None` when nothing + /// changed. + pub fn commit(&self, message: &str) -> Result> { + self.check_branch()?; + self.git(&["add", "-A"])?; + if self.git_status(&["diff", "--cached", "--quiet"])? { + return Ok(None); + } + let name = format!("user.name={}", self.author_name); + let email = format!("user.email={}", self.author_email); + self.git(&["-c", &name, "-c", &email, "commit", "-q", "-m", message])?; + Ok(Some(self.git(&["rev-parse", "HEAD"])?.trim().to_string())) + } + + /// Pushes the working branch. Never forces and never pushes the base. + pub fn push(&self) -> Result<()> { + self.check_branch()?; + let refspec = format!("HEAD:refs/heads/{}", self.branch); + self.remote_git(&self.dir, &["push", "-u", "origin", &refspec]) + .context("pushing the branch (it was not forced)")?; + Ok(()) + } + + /// `git status --short --branch`, for the agent's first message. + pub fn status(&self) -> Result { + self.git(&["status", "--short", "--branch"]) + } + + fn check_branch(&self) -> Result<()> { + let current = self.git(&["rev-parse", "--abbrev-ref", "HEAD"])?; + let current = current.trim(); + if current == self.base { + bail!("refusing to commit or push on the base branch {current:?}"); + } + if current != self.branch { + bail!( + "the working branch changed to {current:?} (expected {:?}); not committing", + self.branch + ); + } + Ok(()) + } + + fn default_branch(&self) -> Result { + let head = self.git(&["symbolic-ref", "--short", "refs/remotes/origin/HEAD"])?; + Ok(head.trim().trim_start_matches("origin/").to_string()) + } + + fn remote_branches(&self) -> Result> { + let out = self.git(&[ + "for-each-ref", + "--format=%(refname:short)", + "refs/remotes/origin", + ])?; + Ok(out + .lines() + .filter_map(|l| l.strip_prefix("origin/")) + .filter(|b| *b != "HEAD") + .map(str::to_string) + .collect()) + } + + fn git(&self, args: &[&str]) -> Result { + let out = self.command(&self.dir, args, false).output()?; + check(out, args) + } + + /// Runs git and reports whether it exited 0 (for `--quiet` checks). + fn git_status(&self, args: &[&str]) -> Result { + Ok(self.command(&self.dir, args, false).status()?.success()) + } + + /// A git command that talks to the remote: gets the token through the + /// askpass helper, and no credential helper may store it. + fn remote_git(&self, cwd: &Path, args: &[&str]) -> Result { + let out = self.command(cwd, args, true).output()?; + check(out, args) + } + + fn command(&self, cwd: &Path, args: &[&str], remote: bool) -> Command { + let mut cmd = Command::new("git"); + cmd.current_dir(cwd) + .env("GIT_TERMINAL_PROMPT", "0") + .env_remove(TOKEN_VAR) + .env_remove("GITAGENT_GIT_TOKEN") + .env_remove("GITHUB_TOKEN"); + if remote { + cmd.args(["-c", "credential.helper="]); + if let (Some(helper), Some(token)) = (&self.askpass, &self.token) { + cmd.env("GIT_ASKPASS", helper).env(TOKEN_VAR, token); + } + } + cmd.args(args); + cmd + } +} + +fn check(out: Output, args: &[&str]) -> Result { + if out.status.success() { + return Ok(String::from_utf8_lossy(&out.stdout).into_owned()); + } + bail!( + "git {} failed: {}", + args.first().unwrap_or(&""), + String::from_utf8_lossy(&out.stderr).trim() + ) +} + +/// The helper prints a user name or the token from the environment. The +/// script itself holds no secret. +fn write_askpass(dir: &Path) -> Result { + use std::os::unix::fs::PermissionsExt; + std::fs::create_dir_all(dir)?; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + let path = dir.join("git-askpass.sh"); + std::fs::write( + &path, + format!( + "#!/bin/sh\ncase \"$1\" in\n Username*) echo x-access-token ;;\n *) printf '%s\\n' \"${TOKEN_VAR}\" ;;\nesac\n" + ), + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700))?; + Ok(path) +} + +fn split_author(author: &str) -> (String, String) { + match author.split_once('<') { + Some((name, rest)) => ( + name.trim().to_string(), + rest.trim_end_matches('>').trim().to_string(), + ), + None => (author.trim().to_string(), "gitagent@localhost".to_string()), + } +} + +/// Lowercase letters, digits and dashes from the start of the task. +pub fn slug(task: &str) -> String { + let mut out = String::new(); + for c in task.chars() { + if c.is_ascii_alphanumeric() { + out.push(c.to_ascii_lowercase()); + } else if !out.ends_with('-') && !out.is_empty() { + out.push('-'); + } + if out.len() >= 40 { + break; + } + } + let out = out.trim_end_matches('-').to_string(); + if out.is_empty() { + "task".into() + } else { + out + } +} + +fn short_id() -> String { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + format!( + "{:06x}", + (nanos ^ u128::from(std::process::id())) & 0xff_ffff + ) +} + +#[cfg(test)] +mod tests; diff --git a/crates/repo/src/tests.rs b/crates/repo/src/tests.rs new file mode 100644 index 00000000..603e03da --- /dev/null +++ b/crates/repo/src/tests.rs @@ -0,0 +1,223 @@ +use super::*; + +/// A bare "remote" with one commit on `main`, as a local path URL. +fn remote() -> (tempfile::TempDir, String) { + let root = tempfile::tempdir().unwrap(); + let bare = root.path().join("origin.git"); + let seed = root.path().join("seed"); + let git = |dir: &Path, args: &[&str]| { + let out = Command::new("git") + .current_dir(dir) + .args(args) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + git( + root.path(), + &["init", "-q", "--bare", "-b", "main", "origin.git"], + ); + git(root.path(), &["init", "-q", "-b", "main", "seed"]); + std::fs::write(seed.join("README.md"), "hello\n").unwrap(); + git(&seed, &["add", "."]); + git( + &seed, + &[ + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "-m", + "init", + ], + ); + git(&seed, &["remote", "add", "origin", &bare.to_string_lossy()]); + git(&seed, &["push", "-q", "origin", "main"]); + (root, bare.to_string_lossy().into_owned()) +} + +fn options(root: &Path, url: &str, name: &str) -> RepoOptions { + RepoOptions { + url: url.into(), + base: None, + branch: None, + dir: root.join(name), + branch_prefix: "gitagent/".into(), + author: "gitagent ".into(), + token: Some("supersecret-token-123".into()), + helper_dir: root.join(format!("{name}-helper")), + } +} + +fn remote_branches(url: &str) -> String { + let out = Command::new("git") + .args(["ls-remote", "--heads", url]) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +#[test] +fn clones_branches_commits_and_pushes() { + let (root, url) = remote(); + let repo = + Repo::clone_for_task(&options(root.path(), &url, "work"), "Fix the failing test!").unwrap(); + assert_eq!(repo.base, "main"); + assert!( + repo.branch.starts_with("gitagent/fix-the-failing-test-"), + "{}", + repo.branch + ); + assert!(!repo.resumed); + + // Nothing changed: no commit. + assert_eq!(repo.commit("gitagent: nothing").unwrap(), None); + + std::fs::write(repo.dir.join("fix.txt"), "fixed\n").unwrap(); + let sha = repo.commit("gitagent: fix").unwrap().expect("a commit"); + assert_eq!(sha.len(), 40); + let log = Command::new("git") + .current_dir(&repo.dir) + .args(["log", "-1", "--format=%an <%ae>|%s"]) + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&log.stdout).trim(), + "gitagent |gitagent: fix" + ); + + repo.push().unwrap(); + let heads = remote_branches(&url); + assert!( + heads.contains(&format!("refs/heads/{}", repo.branch)), + "{heads}" + ); + // main is untouched. + let main_log = Command::new("git") + .args(["--git-dir", &url, "log", "--format=%s", "main"]) + .output() + .unwrap(); + assert_eq!(String::from_utf8_lossy(&main_log.stdout).trim(), "init"); +} + +#[test] +fn resumes_an_existing_branch() { + let (root, url) = remote(); + let mut first = options(root.path(), &url, "first"); + first.branch = Some("feature/x".into()); + let repo = Repo::clone_for_task(&first, "t").unwrap(); + assert!(!repo.resumed); + std::fs::write(repo.dir.join("a.txt"), "a\n").unwrap(); + repo.commit("gitagent: a").unwrap(); + repo.push().unwrap(); + + let mut second = options(root.path(), &url, "second"); + second.branch = Some("feature/x".into()); + let again = Repo::clone_for_task(&second, "t").unwrap(); + assert!(again.resumed); + assert_eq!( + std::fs::read_to_string(again.dir.join("a.txt")).unwrap(), + "a\n" + ); +} + +#[test] +fn refuses_the_base_branch_and_never_forces() { + let (root, url) = remote(); + let mut o = options(root.path(), &url, "base"); + o.branch = Some("main".into()); + let err = Repo::clone_for_task(&o, "t").unwrap_err().to_string(); + assert!(err.contains("refusing to work on the base branch"), "{err}"); + + // Two clones push different histories to the same branch: the second + // push is rejected rather than forced. + let mut a = options(root.path(), &url, "a"); + a.branch = Some("shared".into()); + let ra = Repo::clone_for_task(&a, "t").unwrap(); + let mut b = options(root.path(), &url, "b"); + b.branch = Some("shared".into()); + let rb = Repo::clone_for_task(&b, "t").unwrap(); + std::fs::write(ra.dir.join("a.txt"), "a\n").unwrap(); + ra.commit("a").unwrap(); + ra.push().unwrap(); + std::fs::write(rb.dir.join("b.txt"), "b\n").unwrap(); + rb.commit("b").unwrap(); + assert!(rb.push().is_err()); + + // Committing after the agent switched to the base branch is refused. + Command::new("git") + .current_dir(&ra.dir) + .args(["checkout", "-q", "main"]) + .output() + .unwrap(); + std::fs::write(ra.dir.join("c.txt"), "c\n").unwrap(); + assert!(ra + .commit("c") + .unwrap_err() + .to_string() + .contains("base branch")); +} + +#[test] +fn the_token_never_lands_in_the_repository() { + let (root, url) = remote(); + let repo = Repo::clone_for_task(&options(root.path(), &url, "tok"), "t").unwrap(); + std::fs::write(repo.dir.join("x"), "x\n").unwrap(); + repo.commit("x").unwrap(); + repo.push().unwrap(); + let mut stack = vec![repo.dir.join(".git")]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let p = entry.path(); + if p.is_dir() { + stack.push(p); + } else if let Ok(bytes) = std::fs::read(&p) { + assert!( + !String::from_utf8_lossy(&bytes).contains("supersecret"), + "token found in {}", + p.display() + ); + } + } + } + let helper = std::fs::read_to_string(root.path().join("tok-helper/git-askpass.sh")).unwrap(); + assert!(!helper.contains("supersecret")); +} + +#[test] +fn slugs_and_non_empty_targets() { + assert_eq!(slug("Fix the failing test!"), "fix-the-failing-test"); + assert_eq!(slug(" *** "), "task"); + assert!(slug(&"a".repeat(100)).len() <= 41); + let (root, url) = remote(); + std::fs::create_dir_all(root.path().join("busy")).unwrap(); + std::fs::write(root.path().join("busy/file"), "x").unwrap(); + let err = Repo::clone_for_task(&options(root.path(), &url, "busy"), "t").unwrap_err(); + assert!(err.to_string().contains("not empty")); +} + +#[test] +fn askpass_helper_answers_user_and_password_prompts() { + let dir = tempfile::tempdir().unwrap(); + let helper = write_askpass(&dir.path().join("h")).unwrap(); + let ask = |prompt: &str| { + let out = Command::new("sh") + .arg(&helper) + .arg(prompt) + .env(TOKEN_VAR, "tok-value") + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(ask("Username for 'https://github.com': "), "x-access-token"); + assert_eq!( + ask("Password for 'https://x-access-token@github.com': "), + "tok-value" + ); +} diff --git a/crates/sandbox/Cargo.toml b/crates/sandbox/Cargo.toml new file mode 100644 index 00000000..4bb6ae39 --- /dev/null +++ b/crates/sandbox/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "gitagent-sandbox" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints.rust] +unsafe_code = "deny" + +[dependencies] +anyhow.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +landlock = "0.4.7" +libc = "0.2" +seccompiler = "0.5.0" + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/sandbox/src/lib.rs b/crates/sandbox/src/lib.rs new file mode 100644 index 00000000..44e8a1f8 --- /dev/null +++ b/crates/sandbox/src/lib.rs @@ -0,0 +1,192 @@ +//! Sandbox for the agent's shell (milestone M7), applied by the +//! `gitagent __sandbox` launcher before it executes the shell, so the +//! restrictions cover the shell and everything it starts while gitagent +//! itself stays unrestricted. +//! +//! On Linux: +//! - Landlock: writes are allowed only beneath the given directories (reading +//! is not restricted). +//! - seccomp: a deny-list of system calls an agent never needs (mounting, +//! kernel modules, rebooting, keyrings, BPF, ...); they fail with EPERM. +//! +//! Elsewhere it is not available and nothing is restricted. +//! +//! This crate is the only one allowed to use `unsafe`; it currently needs none. + +use std::path::PathBuf; + +use anyhow::Result; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Status { + /// Landlock and seccomp both applied. + Enforced, + /// Something was applied, with a note on what was not. + Partial(String), + /// Not available on this system. + Unsupported(String), +} + +/// Directories the sandboxed shell may write beneath. +#[derive(Debug, Clone, Default)] +pub struct Policy { + pub writable: Vec, +} + +pub fn available() -> bool { + cfg!(target_os = "linux") +} + +#[cfg(target_os = "linux")] +pub fn apply(policy: &Policy) -> Result { + let landlock = linux::landlock(policy); + let seccomp = linux::seccomp(); + Ok(match (landlock, seccomp) { + (Ok(true), Ok(())) => Status::Enforced, + (Ok(false), Ok(())) => Status::Partial("Landlock is not supported by this kernel".into()), + (Ok(_), Err(e)) => Status::Partial(format!("seccomp filter not applied: {e:#}")), + (Err(e), Ok(())) => Status::Partial(format!("Landlock not applied: {e:#}")), + (Err(a), Err(b)) => Status::Unsupported(format!("Landlock: {a:#}; seccomp: {b:#}")), + }) +} + +#[cfg(not(target_os = "linux"))] +pub fn apply(_policy: &Policy) -> Result { + Ok(Status::Unsupported( + "the sandbox needs Linux (Landlock and seccomp)".into(), + )) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::collections::BTreeMap; + + use anyhow::{Context, Result}; + use landlock::{ + path_beneath_rules, AccessFs, Ruleset, RulesetAttr, RulesetCreatedAttr, RulesetStatus, ABI, + }; + use seccompiler::{SeccompAction, SeccompFilter, TargetArch}; + + use super::Policy; + + /// Returns false when the kernel does not support Landlock. + pub fn landlock(policy: &Policy) -> Result { + let abi = ABI::V5; + let write = AccessFs::from_write(abi); + let existing: Vec<&std::path::Path> = policy + .writable + .iter() + .map(|p| p.as_path()) + .filter(|p| p.exists()) + .collect(); + let status = Ruleset::default() + .handle_access(write)? + .create()? + .add_rules(path_beneath_rules(existing, write))? + .restrict_self() + .context("restricting the process")?; + Ok(status.ruleset != RulesetStatus::NotEnforced) + } + + pub fn seccomp() -> Result<()> { + let mut denied: Vec = vec![ + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_swapon, + libc::SYS_swapoff, + libc::SYS_reboot, + libc::SYS_kexec_load, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_bpf, + libc::SYS_add_key, + libc::SYS_keyctl, + libc::SYS_request_key, + libc::SYS_setns, + libc::SYS_open_by_handle_at, + libc::SYS_userfaultfd, + libc::SYS_acct, + libc::SYS_settimeofday, + libc::SYS_clock_settime, + libc::SYS_sethostname, + libc::SYS_setdomainname, + libc::SYS_quotactl, + libc::SYS_vhangup, + ]; + #[cfg(target_arch = "x86_64")] + denied.extend([libc::SYS_iopl, libc::SYS_ioperm, libc::SYS_kexec_file_load]); + // kexec_file_load; libc's musl table for aarch64 lacks the constant. + #[cfg(target_arch = "aarch64")] + denied.push(294); + + let rules: BTreeMap> = + denied.into_iter().map(|nr| (nr, Vec::new())).collect(); + #[cfg(target_arch = "x86_64")] + let arch = TargetArch::x86_64; + #[cfg(target_arch = "aarch64")] + let arch = TargetArch::aarch64; + let filter = SeccompFilter::new( + rules, + SeccompAction::Allow, + SeccompAction::Errno(libc::EPERM as u32), + arch, + )?; + let program: seccompiler::BpfProgram = filter.try_into()?; + seccompiler::apply_filter(&program)?; + Ok(()) + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use std::process::Command; + + /// Runs a sandboxed child (a re-run of this test binary) and reports + /// what it could write. + #[test] + fn writes_outside_the_allowed_directories_fail() { + if std::env::var("GA_SANDBOX_CHILD").is_ok() { + return; + } + let allowed = tempfile::tempdir().unwrap(); + let other = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + let script = format!( + "echo ok > {a}/in.txt && echo in-ok; echo x > {o}/out.txt 2>/dev/null && echo out-ok || echo out-denied", + a = allowed.path().display(), + o = other.path().display() + ); + let exe = std::env::current_exe().unwrap(); + let out = Command::new(exe) + .args(["--exact", "tests::sandboxed_child", "--nocapture"]) + .env("GA_SANDBOX_CHILD", "1") + .env("GA_ALLOWED", allowed.path()) + .env("GA_SCRIPT", script) + .output() + .unwrap(); + let text = String::from_utf8_lossy(&out.stdout); + if text.contains("status Unsupported") { + eprintln!("Landlock unavailable here; skipping: {text}"); + return; + } + assert!(text.contains("in-ok"), "{text}"); + assert!(text.contains("out-denied"), "{text}"); + } + + #[test] + fn sandboxed_child() { + let Ok(allowed) = std::env::var("GA_ALLOWED") else { + return; + }; + let status = apply(&Policy { + writable: vec![allowed.into(), "/dev".into()], + }) + .unwrap(); + println!("status {status:?}"); + let script = std::env::var("GA_SCRIPT").unwrap(); + let out = Command::new("sh").arg("-c").arg(script).output().unwrap(); + println!("{}", String::from_utf8_lossy(&out.stdout)); + } +} diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml new file mode 100644 index 00000000..7b45796a --- /dev/null +++ b/crates/sdk/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "gitagent-sdk" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +async-trait.workspace = true +gitagent-core.workspace = true +gitagent-hooks.workspace = true +gitagent-loader.workspace = true +gitagent-provider.workspace = true +gitagent-record.workspace = true +gitagent-tools.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs new file mode 100644 index 00000000..16244f95 --- /dev/null +++ b/crates/sdk/src/lib.rs @@ -0,0 +1,999 @@ +//! A library interface over gitagent. +//! +//! `query(prompt, options)` runs one agent loop and streams messages: a +//! `system` / `init` message, then an `assistant` and a `user` message per +//! turn (tool uses and their results), `system` / `compact_boundary` when +//! the history is compacted, and a final `result` with turns, usage and +//! cost. The stream's field names are `session_id`, `parent_tool_use_id`, +//! `num_turns`, `total_cost_usd`, `modelUsage`, …, and tools appear under +//! the `.claude/` tool and argument names (`Bash` / `command`), as they do +//! in `.claude/settings.json` hooks. +//! +//! The loop underneath is gitagent's own (`gitagent_core::run`): the +//! stream shape is the interface, not the implementation. + +use std::collections::BTreeMap; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use gitagent_core::{Outcome, RunOptions, RunReport}; +use gitagent_hooks::{Decision, Matcher, ToolHooks}; +use gitagent_loader::{sdk as names, AgentDef, AgentSpec, LoadOptions}; +use gitagent_record::Events; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use tokio::sync::mpsc; + +// ---------------------------------------------------------------- options + +/// Where settings files are read from. With none, only gitagent's +/// built-in configuration applies. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SettingSource { + /// `~/.gitagent/` and `~/.claude/`. + User, + /// The working directory's `gitagent.yaml`, `.gitagent/` and `.claude/`. + Project, +} + +/// A sub-agent given in code. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", default)] +pub struct AgentDefinition { + pub description: String, + pub prompt: String, + /// `.claude/` or gitagent tool names; `None` inherits the parent's tools. + pub tools: Option>, + pub disallowed_tools: Vec, + pub model: Option, + pub max_turns: Option, + pub effort: Option, +} + +impl AgentDefinition { + fn to_def(&self, name: &str) -> AgentDef { + let general = gitagent_loader::GENERAL_AGENT; + let _ = general; + AgentDef { + name: name.to_string(), + description: self.description.clone(), + tools: self.tools.as_ref().map(|t| names::normalize_tool_list(t)), + disallowed_tools: names::normalize_tool_list(&self.disallowed_tools), + max_turns: self.max_turns.unwrap_or(30), + model: self.model.clone().filter(|m| m != "inherit"), + effort: self.effort.clone(), + instructions: self.prompt.clone(), + } + } +} + +/// Prices per million tokens, for `total_cost_usd`. Without them the cost +/// is reported as 0. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Prices { + pub input: f64, + pub output: f64, + pub cache_read: f64, + pub cache_write: f64, +} + +/// Options for `query`. +#[derive(Clone, Default)] +pub struct AgentOptions { + /// `provider:model` (or a bare model id for Anthropic). Overrides the config. + pub model: Option, + /// Replaces the built-in system prompt. + pub system_prompt: Option, + /// Added after the system prompt. + pub append_system_prompt: Option, + /// The agent's working directory (default: the current directory). + pub cwd: Option, + /// `.claude/` or gitagent tool names; empty means every tool. + pub allowed_tools: Vec, + pub disallowed_tools: Vec, + pub max_turns: Option, + /// Wall-clock budget (default: `timeout_s` from the config). + pub timeout: Option, + /// Sub-agents by name. Given agents turn the sub-agent tool on. + pub agents: BTreeMap, + /// Hook callbacks. + pub hooks: Vec, + /// Which settings files to read; the default is none. + pub setting_sources: Vec, + /// gitagent-specific: a profile name or path, a `gitagent.yaml`, where + /// records go, and prices for the cost estimate. + pub profile: Option, + pub config_path: Option, + pub out_dir: Option, + pub prices: Option, +} + +// ---------------------------------------------------------------- hooks + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HookEvent { + PreToolUse, + PostToolUse, +} + +/// What a hook callback receives, with `.claude/` tool and argument +/// names. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct HookInput { + pub session_id: String, + pub cwd: String, + pub hook_event_name: String, + pub tool_name: String, + pub tool_input: Value, + pub tool_use_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_response: Option, +} + +/// What a hook callback returns (the parts gitagent acts on). +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", default)] +pub struct HookOutput { + #[serde(rename = "continue", skip_serializing_if = "Option::is_none")] + pub continue_: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_reason: Option, + /// `"block"` blocks; the other value is `"approve"`. + #[serde(skip_serializing_if = "Option::is_none")] + pub decision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hook_specific_output: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", default)] +pub struct HookSpecificOutput { + #[serde(skip_serializing_if = "Option::is_none")] + pub hook_event_name: Option, + /// `allow`, `deny` or `ask` (treated as deny: nobody can be asked). + #[serde(skip_serializing_if = "Option::is_none")] + pub permission_decision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub permission_decision_reason: Option, + /// New arguments, in `.claude/` argument names. + #[serde(skip_serializing_if = "Option::is_none")] + pub updated_input: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub updated_tool_output: Option, +} + +impl HookOutput { + pub fn allow() -> Self { + HookOutput::default() + } + + pub fn deny(reason: impl Into) -> Self { + HookOutput { + hook_specific_output: Some(HookSpecificOutput { + permission_decision: Some("deny".into()), + permission_decision_reason: Some(reason.into()), + ..Default::default() + }), + ..Default::default() + } + } + + pub fn updated_input(input: Map) -> Self { + HookOutput { + hook_specific_output: Some(HookSpecificOutput { + permission_decision: Some("allow".into()), + updated_input: Some(input), + ..Default::default() + }), + ..Default::default() + } + } + + pub fn updated_tool_output(text: impl Into) -> Self { + HookOutput { + hook_specific_output: Some(HookSpecificOutput { + updated_tool_output: Some(text.into()), + ..Default::default() + }), + ..Default::default() + } + } +} + +pub type HookFuture = Pin + Send>>; +pub type HookCallback = Arc HookFuture + Send + Sync>; + +/// Callbacks for one event, for the tools a matcher selects. +#[derive(Clone)] +pub struct HookMatcher { + pub event: HookEvent, + /// `*` or `None` for every tool, exact names (`Edit|Write`), or a regex. + pub matcher: Option, + pub hooks: Vec, +} + +impl HookMatcher { + /// A matcher from a plain async function. + pub fn new(event: HookEvent, matcher: Option<&str>, f: F) -> Self + where + F: Fn(HookInput) -> Fut + Send + Sync + 'static, + Fut: Future + Send + 'static, + { + HookMatcher { + event, + matcher: matcher.map(str::to_string), + hooks: vec![Arc::new(move |input| Box::pin(f(input)))], + } + } +} + +struct CallbackHooks { + matchers: Vec<(Matcher, HookMatcher)>, + cwd: String, + session_id: String, +} + +impl CallbackHooks { + fn new(hooks: &[HookMatcher], cwd: &Path, session_id: &str) -> Option { + let matchers: Vec<(Matcher, HookMatcher)> = hooks + .iter() + .filter_map(|h| Matcher::parse(h.matcher.as_deref()).map(|m| (m, h.clone()))) + .collect(); + (!matchers.is_empty()).then(|| CallbackHooks { + matchers, + cwd: cwd.to_string_lossy().into_owned(), + session_id: session_id.to_string(), + }) + } + + fn input( + &self, + event: HookEvent, + tool: &str, + args: &Map, + response: Option, + ) -> HookInput { + HookInput { + session_id: self.session_id.clone(), + cwd: self.cwd.clone(), + hook_event_name: match event { + HookEvent::PreToolUse => "PreToolUse".into(), + HookEvent::PostToolUse => "PostToolUse".into(), + }, + tool_name: names::sdk_name(tool).to_string(), + tool_input: Value::Object(names::to_sdk_input(tool, args)), + tool_use_id: String::new(), + tool_response: response, + } + } + + /// The rules for one answer. + fn interpret(event: HookEvent, out: HookOutput) -> Decision { + if out.continue_ == Some(false) { + return Decision::Block { + reason: out + .stop_reason + .unwrap_or_else(|| "stopped by a hook".into()), + }; + } + if out.decision.as_deref() == Some("block") { + return Decision::Block { + reason: out.reason.unwrap_or_else(|| "blocked by a hook".into()), + }; + } + let Some(specific) = out.hook_specific_output else { + return Decision::Allow; + }; + match event { + HookEvent::PreToolUse => { + if matches!( + specific.permission_decision.as_deref(), + Some("deny") | Some("ask") + ) { + return Decision::Block { + reason: specific + .permission_decision_reason + .unwrap_or_else(|| "denied by a hook".into()), + }; + } + match specific.updated_input { + Some(args) => Decision::Modify(Value::Object(args)), + None => Decision::Allow, + } + } + HookEvent::PostToolUse => match specific.updated_tool_output { + Some(text) => Decision::Modify(Value::String(text)), + None => Decision::Allow, + }, + } + } +} + +#[async_trait::async_trait] +impl ToolHooks for CallbackHooks { + async fn pre(&self, tool: &str, arguments: &Map) -> Decision { + let mut args = arguments.clone(); + let mut changed = false; + for (matcher, h) in &self.matchers { + if h.event != HookEvent::PreToolUse || !matcher.matches(tool) { + continue; + } + for cb in &h.hooks { + let input = self.input(HookEvent::PreToolUse, tool, &args, None); + match Self::interpret(HookEvent::PreToolUse, cb(input).await) { + Decision::Allow => {} + Decision::Modify(Value::Object(sdk_args)) => { + args = names::from_sdk_input(tool, &sdk_args); + changed = true; + } + other => return other, + } + } + } + if changed { + Decision::Modify(Value::Object(args)) + } else { + Decision::Allow + } + } + + async fn post( + &self, + tool: &str, + arguments: &Map, + output: &str, + is_error: bool, + ) -> Decision { + let mut out = output.to_string(); + let mut changed = false; + for (matcher, h) in &self.matchers { + if h.event != HookEvent::PostToolUse || !matcher.matches(tool) { + continue; + } + for cb in &h.hooks { + let response = json!({"output": out, "is_error": is_error}); + let input = self.input(HookEvent::PostToolUse, tool, arguments, Some(response)); + match Self::interpret(HookEvent::PostToolUse, cb(input).await) { + Decision::Allow => {} + Decision::Modify(Value::String(text)) => { + out = text; + changed = true; + } + other => return other, + } + } + } + if changed { + Decision::Modify(Value::String(out)) + } else { + Decision::Allow + } + } +} + +// ---------------------------------------------------------------- messages + +/// A content block as the Messages API writes it. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentBlock { + Text { + text: String, + }, + ToolUse { + id: String, + name: String, + input: Value, + }, + ToolResult { + tool_use_id: String, + content: String, + is_error: bool, + }, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct Usage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_input_tokens: u64, + pub cache_creation_input_tokens: u64, +} + +/// Per-model usage (camelCase, as the stream writes it). +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ModelUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_input_tokens: u64, + pub cache_creation_input_tokens: u64, + #[serde(rename = "costUSD")] + pub cost_usd: f64, + pub context_window: u64, +} + +/// The API message inside an assistant or user message. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ApiMessage { + pub role: String, + pub content: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CompactMetadata { + pub trigger: String, + pub pre_tokens: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PermissionDenial { + pub tool_name: String, + pub tool_use_id: String, + pub tool_input: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// One message of the stream, tagged by `type`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Message { + System { + subtype: String, + uuid: String, + session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + cwd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tools: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + agents: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + skills: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(rename = "permissionMode")] + permission_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compact_metadata: Option, + }, + Assistant { + uuid: String, + session_id: String, + parent_tool_use_id: Option, + message: ApiMessage, + }, + User { + uuid: String, + session_id: String, + parent_tool_use_id: Option, + message: ApiMessage, + }, + Result { + subtype: String, + uuid: String, + session_id: String, + duration_ms: u64, + duration_api_ms: u64, + is_error: bool, + num_turns: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + result: Option, + stop_reason: Option, + total_cost_usd: f64, + usage: Usage, + #[serde(rename = "modelUsage")] + model_usage: BTreeMap, + permission_denials: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + errors: Vec, + }, +} + +impl Message { + pub fn is_result(&self) -> bool { + matches!(self, Message::Result { .. }) + } +} + +// ---------------------------------------------------------------- converter + +/// Turns gitagent's events (`--events jsonl`) into the stream's messages. +pub struct Converter { + session_id: String, + model: String, + context_window: u64, + prices: Option, + started: Instant, + api_ms: u64, + turn_started: Option, + text: String, + tool_uses: Vec, + tool_results: Vec, + last_call: Option<(String, String, Value)>, + denials: Vec, + usage: Usage, + cost: f64, + seq: u64, +} + +impl Converter { + pub fn new(session_id: &str, model: &str, context_window: u64, prices: Option) -> Self { + Converter { + session_id: session_id.to_string(), + model: model.to_string(), + context_window, + prices, + started: Instant::now(), + api_ms: 0, + turn_started: None, + text: String::new(), + tool_uses: Vec::new(), + tool_results: Vec::new(), + last_call: None, + denials: Vec::new(), + usage: Usage::default(), + cost: 0.0, + seq: 0, + } + } + + fn uuid(&mut self) -> String { + self.seq += 1; + format!("{}-{:04}", self.session_id, self.seq) + } + + /// The opening `system` / `init` message. + pub fn init( + &mut self, + cwd: &Path, + tools: &[String], + agents: &[String], + skills: &[String], + ) -> Message { + Message::System { + subtype: "init".into(), + uuid: self.uuid(), + session_id: self.session_id.clone(), + cwd: Some(cwd.to_string_lossy().into_owned()), + model: Some(self.model.clone()), + tools: tools + .iter() + .map(|t| names::sdk_name(t).to_string()) + .collect(), + agents: agents.to_vec(), + skills: skills.to_vec(), + permission_mode: Some("bypassPermissions".into()), + compact_metadata: None, + } + } + + /// Messages produced by one event, in order. + pub fn feed(&mut self, event: &Value) -> Vec { + let kind = event["event"].as_str().unwrap_or(""); + match kind { + "turn_start" => { + self.turn_started = Some(Instant::now()); + self.text.clear(); + self.tool_uses.clear(); + self.tool_results.clear(); + Vec::new() + } + "model_text" => { + if let Some(t) = event["text"].as_str() { + self.text = t.to_string(); + } + if let Some(m) = event["model"].as_str() { + self.model = m.to_string(); + } + if let Some(start) = self.turn_started { + self.api_ms += start.elapsed().as_millis() as u64; + } + Vec::new() + } + "tool_call" => { + let id = event["id"].as_str().unwrap_or("").to_string(); + let name = event["name"].as_str().unwrap_or("").to_string(); + let args = event["arguments"].as_object().cloned().unwrap_or_default(); + let input = Value::Object(names::to_sdk_input(&name, &args)); + self.last_call = Some((id.clone(), name.clone(), input.clone())); + self.tool_uses.push(ContentBlock::ToolUse { + id, + name: names::sdk_name(&name).to_string(), + input, + }); + Vec::new() + } + "tool_result" => { + self.tool_results.push(ContentBlock::ToolResult { + tool_use_id: event["id"].as_str().unwrap_or("").to_string(), + content: event["content"].as_str().unwrap_or("").to_string(), + is_error: event["is_error"].as_bool().unwrap_or(false), + }); + Vec::new() + } + "guard" => match event["kind"].as_str() { + Some("hook_block") => { + if let Some((id, name, input)) = &self.last_call { + self.denials.push(PermissionDenial { + tool_name: names::sdk_name(name).to_string(), + tool_use_id: id.clone(), + tool_input: input.clone(), + reason: event["reason"].as_str().map(str::to_string), + }); + } + Vec::new() + } + Some("compaction") => vec![Message::System { + subtype: "compact_boundary".into(), + uuid: self.uuid(), + session_id: self.session_id.clone(), + cwd: None, + model: None, + tools: Vec::new(), + agents: Vec::new(), + skills: Vec::new(), + permission_mode: None, + compact_metadata: Some(CompactMetadata { + trigger: "auto".into(), + pre_tokens: event["pre_tokens"].as_u64().unwrap_or(0), + }), + }], + _ => Vec::new(), + }, + "turn_end" => { + let u = &event["usage"]; + let turn = Usage { + input_tokens: u["input_tokens"].as_u64().unwrap_or(0), + output_tokens: u["output_tokens"].as_u64().unwrap_or(0), + cache_read_input_tokens: u["cache_read_tokens"].as_u64().unwrap_or(0), + cache_creation_input_tokens: u["cache_write_tokens"].as_u64().unwrap_or(0), + }; + self.usage.input_tokens += turn.input_tokens; + self.usage.output_tokens += turn.output_tokens; + self.usage.cache_read_input_tokens += turn.cache_read_input_tokens; + self.usage.cache_creation_input_tokens += turn.cache_creation_input_tokens; + self.cost += cost_of(self.prices, &turn); + let mut content = Vec::new(); + if !self.text.is_empty() { + content.push(ContentBlock::Text { + text: std::mem::take(&mut self.text), + }); + } + content.append(&mut self.tool_uses); + let stop_reason = if content + .iter() + .any(|c| matches!(c, ContentBlock::ToolUse { .. })) + { + "tool_use" + } else { + "end_turn" + }; + let mut out = vec![Message::Assistant { + uuid: self.uuid(), + session_id: self.session_id.clone(), + parent_tool_use_id: None, + message: ApiMessage { + role: "assistant".into(), + content, + model: Some(self.model.clone()), + stop_reason: Some(stop_reason.into()), + usage: Some(turn), + }, + }]; + if !self.tool_results.is_empty() { + out.push(Message::User { + uuid: self.uuid(), + session_id: self.session_id.clone(), + parent_tool_use_id: None, + message: ApiMessage { + role: "user".into(), + content: std::mem::take(&mut self.tool_results), + model: None, + stop_reason: None, + usage: None, + }, + }); + } + out + } + "run_end" => { + let outcome = event["outcome"].as_str().unwrap_or(""); + let (subtype, is_error) = match outcome { + "completed" => ("success", false), + "max_turns" => ("error_max_turns", true), + _ => ("error_during_execution", true), + }; + let mut model_usage = BTreeMap::new(); + model_usage.insert( + self.model.clone(), + ModelUsage { + input_tokens: self.usage.input_tokens, + output_tokens: self.usage.output_tokens, + cache_read_input_tokens: self.usage.cache_read_input_tokens, + cache_creation_input_tokens: self.usage.cache_creation_input_tokens, + cost_usd: usd(self.cost), + context_window: self.context_window, + }, + ); + vec![Message::Result { + subtype: subtype.into(), + uuid: self.uuid(), + session_id: self.session_id.clone(), + duration_ms: self.started.elapsed().as_millis() as u64, + duration_api_ms: self.api_ms, + is_error, + num_turns: event["turns"].as_u64().unwrap_or(0) as u32, + result: event["summary"].as_str().map(str::to_string), + stop_reason: (!is_error).then(|| "end_turn".to_string()), + total_cost_usd: usd(self.cost), + usage: self.usage.clone(), + model_usage, + permission_denials: std::mem::take(&mut self.denials), + errors: if is_error { + vec![outcome.to_string()] + } else { + Vec::new() + }, + }] + } + _ => Vec::new(), + } + } +} + +/// Dollars to the cent's hundredth, so costs survive a JSON round trip unchanged. +fn usd(x: f64) -> f64 { + (x * 1e6).round() / 1e6 +} + +fn cost_of(prices: Option, u: &Usage) -> f64 { + let Some(p) = prices else { + return 0.0; + }; + let uncached = u + .input_tokens + .saturating_sub(u.cache_read_input_tokens) + .saturating_sub(u.cache_creation_input_tokens); + (uncached as f64 * p.input + + u.cache_read_input_tokens as f64 * p.cache_read + + u.cache_creation_input_tokens as f64 * p.cache_write + + u.output_tokens as f64 * p.output) + / 1e6 +} + +// ---------------------------------------------------------------- query + +/// A running query: messages arrive as the agent works; `finish` waits for +/// the run and returns gitagent's own report. +pub struct Query { + rx: mpsc::UnboundedReceiver, + handle: tokio::task::JoinHandle>, +} + +impl Query { + /// The next message, or `None` once the stream has ended. + pub async fn next(&mut self) -> Option { + self.rx.recv().await + } + + /// Every remaining message. + pub async fn collect(&mut self) -> Vec { + let mut out = Vec::new(); + while let Some(m) = self.rx.recv().await { + out.push(m); + } + out + } + + /// Waits for the run to end. + pub async fn finish(self) -> Result { + self.handle.await.context("the agent task panicked")? + } +} + +/// The loaded configuration and the run options an `AgentOptions` asks for. +/// Public so the CLI can apply the same options. +pub struct Prepared { + pub spec: AgentSpec, + pub work_dir: PathBuf, + pub out_dir: PathBuf, + pub timeout: Duration, + pub tool_allowlist: Option>, + pub tool_denylist: Vec, + pub hook_callbacks: Option>, + pub session_id: String, +} + +pub fn prepare(options: &AgentOptions) -> Result { + let work_dir = match &options.cwd { + Some(d) => d.clone(), + None => std::env::current_dir().context("reading the current directory")?, + }; + let project = options.setting_sources.contains(&SettingSource::Project); + let user = options.setting_sources.contains(&SettingSource::User); + let mut spec = gitagent_loader::load(&LoadOptions { + config_path: options.config_path.clone(), + model: options.model.clone().map(|m| { + if m.contains(':') { + m + } else { + format!("anthropic:{m}") + } + }), + profile: options.profile.clone(), + project_dir: project.then(|| work_dir.clone()), + home_dir: if user { + std::env::var_os("HOME").map(PathBuf::from) + } else { + None + }, + })?; + if let Some(p) = &options.system_prompt { + spec.system_prompt = format!("{}\n", p.trim_end()); + } + if let Some(extra) = &options.append_system_prompt { + spec.system_prompt.push('\n'); + spec.system_prompt.push_str(extra.trim_end()); + spec.system_prompt.push('\n'); + } + if let Some(n) = options.max_turns { + spec.config.max_turns = n; + } + if !options.agents.is_empty() { + spec.profile.tools.subagents = true; + for (name, def) in &options.agents { + spec.agents.retain(|a| a.name != *name); + spec.agents.push(def.to_def(name)); + } + if !spec + .agents + .iter() + .any(|a| a.name == gitagent_loader::GENERAL_AGENT) + { + // The general-purpose agent stays alongside the defined ones. + spec.agents.push(gitagent_loader::AgentDef { + name: gitagent_loader::GENERAL_AGENT.into(), + description: "A helper for a self-contained piece of work.".into(), + tools: None, + disallowed_tools: Vec::new(), + max_turns: 30, + model: None, + effort: None, + instructions: String::new(), + }); + } + spec.agents.sort_by(|a, b| a.name.cmp(&b.name)); + } + let session_id = format!("sdk-{}", std::process::id()); + let out_dir = options + .out_dir + .clone() + .unwrap_or_else(|| std::env::temp_dir().join("gitagent").join(&session_id)); + let timeout = options + .timeout + .unwrap_or(Duration::from_secs(spec.config.timeout_s)); + let tool_allowlist = (!options.allowed_tools.is_empty()) + .then(|| names::normalize_tool_list(&options.allowed_tools)); + let tool_denylist = names::normalize_tool_list(&options.disallowed_tools); + let hook_callbacks: Option> = + CallbackHooks::new(&options.hooks, &work_dir, &session_id) + .map(|h| Arc::new(h) as Arc); + Ok(Prepared { + spec, + work_dir, + out_dir, + timeout, + tool_allowlist, + tool_denylist, + hook_callbacks, + session_id, + }) +} + +/// Runs one task and streams its messages. +pub async fn query(prompt: impl Into, options: AgentOptions) -> Result { + let prompt = prompt.into(); + let p = prepare(&options)?; + let provider = gitagent_provider::for_spec(&p.spec)?; + let (tx, rx) = mpsc::unbounded_channel::(); + + let converter = Arc::new(Mutex::new(Converter::new( + &p.session_id, + &p.spec.model.to_string(), + u64::from(p.spec.profile.context_window), + options.prices, + ))); + let tool_names: Vec = gitagent_tools_offered(&p); + let init = converter.lock().unwrap().init( + &p.work_dir, + &tool_names, + &p.spec + .agents + .iter() + .map(|a| a.name.clone()) + .collect::>(), + &p.spec + .skills + .iter() + .map(|s| s.name.clone()) + .collect::>(), + ); + let _ = tx.send(init); + let events = { + let tx = tx.clone(); + let converter = converter.clone(); + Events::to_callback(move |event| { + let messages = converter.lock().unwrap().feed(&event); + for m in messages { + let _ = tx.send(m); + } + }) + }; + let run_opts = RunOptions { + instruction: prompt, + work_dir: p.work_dir.clone(), + out_dir: p.out_dir.clone(), + timeout: Some(p.timeout), + session_id: Some(p.session_id.clone()), + events, + resume: None, + sandbox: None, + tool_allowlist: p.tool_allowlist.clone(), + tool_denylist: p.tool_denylist.clone(), + hook_callbacks: p.hook_callbacks.clone(), + }; + let spec = p.spec; + let handle = tokio::spawn(async move { + let finished = gitagent_core::run(&spec, provider.as_ref(), run_opts).await?; + drop(tx); + Ok(finished.report) + }); + Ok(Query { rx, handle }) +} + +/// Tool names after the allow and deny lists, for the init message. +fn gitagent_tools_offered(p: &Prepared) -> Vec { + let mut tools: Vec = + gitagent_tools::specs(&p.spec.profile.tools, p.spec.profile.edit_tool) + .into_iter() + .map(|t| t.name) + .collect(); + if !p.spec.skills.is_empty() { + tools.push(gitagent_tools::SKILL.into()); + } + if !p.spec.agents.is_empty() { + tools.push(gitagent_tools::SUBAGENT.into()); + } + if let Some(allow) = &p.tool_allowlist { + tools.retain(|t| allow.contains(t) || t == gitagent_tools::TASK_COMPLETE); + } + tools.retain(|t| !p.tool_denylist.contains(t) || t == gitagent_tools::TASK_COMPLETE); + tools +} + +/// The outcome a result message reports, for callers that want gitagent's enum. +pub fn outcome_of(subtype: &str) -> Outcome { + match subtype { + "success" => Outcome::Completed, + "error_max_turns" => Outcome::MaxTurns, + other => Outcome::ProviderFailed(other.to_string()), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/sdk/src/tests.rs b/crates/sdk/src/tests.rs new file mode 100644 index 00000000..b041de48 --- /dev/null +++ b/crates/sdk/src/tests.rs @@ -0,0 +1,349 @@ +use super::*; + +fn scripted(dir: &Path, name: &str, yaml: &str) -> String { + let path = dir.join(name); + std::fs::write(&path, yaml).unwrap(); + format!("scripted:{}", path.display()) +} + +const ECHO_THEN_COMPLETE: &str = " +- text: Checking the shell. + tool_calls: + - name: exec + arguments: {cmd: echo hi} +- tool_calls: + - name: task_complete + arguments: {summary: printed hi} +"; + +#[tokio::test] +async fn query_streams_sdk_shaped_messages() { + let dir = tempfile::tempdir().unwrap(); + let model = scripted(dir.path(), "s.yaml", ECHO_THEN_COMPLETE); + let mut q = query( + "Print hi.", + AgentOptions { + model: Some(model), + cwd: Some(dir.path().to_path_buf()), + out_dir: Some(dir.path().join("out")), + prices: Some(Prices { + input: 4.0, + output: 20.0, + cache_read: 0.2, + cache_write: 5.0, + }), + ..Default::default() + }, + ) + .await + .unwrap(); + let messages = q.collect().await; + let report = q.finish().await.unwrap(); + assert_eq!(report.outcome, Outcome::Completed); + + let types: Vec<&str> = messages + .iter() + .map(|m| match m { + Message::System { subtype, .. } => { + if subtype == "init" { + "init" + } else { + "system" + } + } + Message::Assistant { .. } => "assistant", + Message::User { .. } => "user", + Message::Result { .. } => "result", + }) + .collect(); + assert_eq!( + types, + vec!["init", "assistant", "user", "assistant", "user", "result"] + ); + + // The init message names the tools with `.claude/` names. + let Message::System { + tools, + cwd, + permission_mode, + .. + } = &messages[0] + else { + panic!() + }; + assert!(tools.contains(&"Bash".to_string()) && tools.contains(&"task_complete".to_string())); + assert_eq!(cwd.as_deref(), Some(dir.path().to_string_lossy().as_ref())); + assert_eq!(permission_mode.as_deref(), Some("bypassPermissions")); + + // Turn 1: text plus a Bash tool_use with `.claude/` argument names, then its result. + let Message::Assistant { + message, + parent_tool_use_id, + .. + } = &messages[1] + else { + panic!() + }; + assert_eq!(parent_tool_use_id, &None); + assert_eq!(message.role, "assistant"); + assert_eq!(message.stop_reason.as_deref(), Some("tool_use")); + assert_eq!( + message.content[0], + ContentBlock::Text { + text: "Checking the shell.".into() + } + ); + let ContentBlock::ToolUse { id, name, input } = &message.content[1] else { + panic!() + }; + assert_eq!(name, "Bash"); + assert_eq!(input, &json!({"command": "echo hi"})); + let Message::User { message: user, .. } = &messages[2] else { + panic!() + }; + let ContentBlock::ToolResult { + tool_use_id, + content, + is_error, + } = &user.content[0] + else { + panic!() + }; + assert_eq!((tool_use_id, *is_error), (id, false)); + assert!(content.contains("hi"), "{content}"); + + // The result carries turns, usage, cost and the summary. + let Message::Result { + subtype, + is_error, + num_turns, + result, + total_cost_usd, + usage, + model_usage, + stop_reason, + .. + } = messages.last().unwrap() + else { + panic!() + }; + assert_eq!( + (subtype.as_str(), *is_error, *num_turns), + ("success", false, 2) + ); + assert_eq!(result.as_deref(), Some("printed hi")); + assert_eq!(stop_reason.as_deref(), Some("end_turn")); + assert!(usage.input_tokens > 0 && *total_cost_usd > 0.0); + assert_eq!(model_usage.len(), 1); + + // The JSON uses the stream's field names. + let text = serde_json::to_string(messages.last().unwrap()).unwrap(); + assert!( + text.contains("\"type\":\"result\"") + && text.contains("\"modelUsage\"") + && text.contains("\"num_turns\"") + ); + let round: Message = serde_json::from_str(&text).unwrap(); + assert_eq!(&round, messages.last().unwrap()); +} + +#[tokio::test] +async fn hook_callbacks_and_tool_lists_apply() { + let dir = tempfile::tempdir().unwrap(); + let script = " +- tool_calls: + - name: exec + arguments: {cmd: rm -rf /tmp/x} +- tool_calls: + - name: exec + arguments: {cmd: echo ok} +- tool_calls: + - name: task_complete + arguments: {summary: done} +"; + let model = scripted(dir.path(), "s.yaml", script); + let seen = Arc::new(Mutex::new(Vec::::new())); + let seen2 = seen.clone(); + let pre = HookMatcher::new( + HookEvent::PreToolUse, + Some("Bash"), + move |input: HookInput| { + seen2.lock().unwrap().push(input.clone()); + async move { + let cmd = input.tool_input["command"].as_str().unwrap_or(""); + if cmd.starts_with("rm -rf") { + HookOutput::deny("no rm -rf") + } else if cmd == "echo ok" { + let mut m = Map::new(); + m.insert("command".into(), json!("echo changed")); + HookOutput::updated_input(m) + } else { + HookOutput::allow() + } + } + }, + ); + let post = HookMatcher::new(HookEvent::PostToolUse, None, |_input: HookInput| async { + HookOutput::updated_tool_output("[output hidden by a hook]") + }); + let mut q = query( + "x", + AgentOptions { + model: Some(model), + cwd: Some(dir.path().to_path_buf()), + out_dir: Some(dir.path().join("out")), + allowed_tools: vec!["Bash".into(), "Read".into()], + disallowed_tools: vec!["Read".into()], + hooks: vec![pre, post], + max_turns: Some(5), + ..Default::default() + }, + ) + .await + .unwrap(); + let messages = q.collect().await; + q.finish().await.unwrap(); + + let Message::System { tools, .. } = &messages[0] else { + panic!() + }; + assert_eq!( + tools, + &vec!["Bash".to_string(), "task_complete".to_string()] + ); + + // The hook saw `.claude/` tool names and argument names. + let inputs = seen.lock().unwrap(); + assert_eq!(inputs[0].tool_name, "Bash"); + assert_eq!(inputs[0].hook_event_name, "PreToolUse"); + assert_eq!(inputs[0].tool_input["command"], "rm -rf /tmp/x"); + + // Turn 1 was denied: the result names the denial; turn 2 ran the changed + // command and its output was replaced by the post hook. + let results: Vec<&ContentBlock> = messages + .iter() + .filter_map(|m| match m { + Message::User { message, .. } => Some(&message.content[0]), + _ => None, + }) + .collect(); + let ContentBlock::ToolResult { + content, is_error, .. + } = results[0] + else { + panic!() + }; + assert!(*is_error && content.contains("no rm -rf"), "{content}"); + let ContentBlock::ToolResult { content, .. } = results[1] else { + panic!() + }; + assert_eq!(content, "[output hidden by a hook]"); + let Message::Result { + permission_denials, .. + } = messages.last().unwrap() + else { + panic!() + }; + assert_eq!(permission_denials.len(), 1); + assert_eq!(permission_denials[0].tool_name, "Bash"); + assert_eq!(permission_denials[0].reason.as_deref(), Some("no rm -rf")); +} + +#[tokio::test] +async fn agents_option_defines_subagents_and_the_system_prompt_can_be_set() { + let dir = tempfile::tempdir().unwrap(); + let model = scripted( + dir.path(), + "s.yaml", + "- tool_calls: [{name: task_complete, arguments: {summary: x}}]\n", + ); + let mut agents = BTreeMap::new(); + agents.insert( + "code-reviewer".to_string(), + AgentDefinition { + description: "Reviews code.".into(), + prompt: "You review.".into(), + tools: Some(vec!["Read".into(), "Grep".into()]), + max_turns: Some(4), + ..Default::default() + }, + ); + let p = prepare(&AgentOptions { + model: Some(model.clone()), + cwd: Some(dir.path().to_path_buf()), + system_prompt: Some("Be brief.".into()), + append_system_prompt: Some("Always verify.".into()), + agents, + ..Default::default() + }) + .unwrap(); + assert_eq!(p.spec.system_prompt, "Be brief.\n\nAlways verify.\n"); + assert!(p.spec.profile.tools.subagents); + let names: Vec<&str> = p.spec.agents.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, vec!["code-reviewer", "general"]); + let r = &p.spec.agents[0]; + assert_eq!( + r.tools.as_deref(), + Some(&["read_file".to_string(), "image_read".to_string()][..]) + ); + assert_eq!((r.max_turns, r.instructions.as_str()), (4, "You review.")); + + // A bare model id means Anthropic. + let p2 = prepare(&AgentOptions { + model: Some("claude-opus-5-5".into()), + cwd: Some(dir.path().to_path_buf()), + ..Default::default() + }) + .unwrap(); + assert_eq!(p2.spec.model.to_string(), "anthropic:claude-opus-5-5"); + + let mut q = query( + "x", + AgentOptions { + model: Some(model), + cwd: Some(dir.path().to_path_buf()), + out_dir: Some(dir.path().join("out")), + ..Default::default() + }, + ) + .await + .unwrap(); + let messages = q.collect().await; + assert!(messages.last().unwrap().is_result()); +} + +#[test] +fn converter_handles_failed_runs_and_compaction() { + let mut c = Converter::new("s", "m", 1000, None); + let boundary = c.feed(&json!({"event": "guard", "kind": "compaction", "pre_tokens": 900})); + let Message::System { + subtype, + compact_metadata, + .. + } = &boundary[0] + else { + panic!() + }; + assert_eq!(subtype, "compact_boundary"); + assert_eq!(compact_metadata.as_ref().unwrap().pre_tokens, 900); + let result = + c.feed(&json!({"event": "run_end", "outcome": "max_turns", "turns": 7, "summary": null})); + let Message::Result { + subtype, + is_error, + num_turns, + errors, + result, + .. + } = &result[0] + else { + panic!() + }; + assert_eq!( + (subtype.as_str(), *is_error, *num_turns), + ("error_max_turns", true, 7) + ); + assert_eq!(errors, &vec!["max_turns".to_string()]); + assert_eq!(result, &None); + assert_eq!(outcome_of("error_max_turns"), Outcome::MaxTurns); +} diff --git a/crates/shaper/Cargo.toml b/crates/shaper/Cargo.toml new file mode 100644 index 00000000..48217f37 --- /dev/null +++ b/crates/shaper/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "gitagent-shaper" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/shaper/src/lib.rs b/crates/shaper/src/lib.rs new file mode 100644 index 00000000..acce9c82 --- /dev/null +++ b/crates/shaper/src/lib.rs @@ -0,0 +1,322 @@ +//! Tool output shaping (BLUEPRINT §5): the full output always goes to a spill +//! file first; the model sees a cleaned, collapsed and capped version that +//! says where the full output is. + +use std::path::{Path, PathBuf}; + +/// Shaping limits, normally taken from the profile's `shape` section. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ShapeCfg { + /// Approximate token budget for one result. + pub cap_tokens: u32, + /// Share of the budget kept from the start; the rest comes from the end. + pub head_ratio: f32, + /// Collapse runs of at least this many similar lines; 0 turns collapsing off. + pub collapse_min_run: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Shaped { + pub text: String, + /// Where the full raw output was saved (absent if writing it failed). + pub spill_path: Option, + /// Lines in the cleaned output, before collapsing and capping. + pub total_lines: usize, + /// Lines the model does not see (collapsed or cut). + pub hidden_lines: usize, + /// Runs of similar lines that were collapsed. + pub collapsed_runs: usize, + pub binary: bool, +} + +/// Shapes successive tool outputs, numbering their spill files. +pub struct Shaper { + pub cfg: ShapeCfg, + pub spill_dir: PathBuf, + next: u64, +} + +impl Shaper { + pub fn new(cfg: ShapeCfg, spill_dir: impl Into) -> Self { + Shaper { + cfg, + spill_dir: spill_dir.into(), + next: 1, + } + } + + pub fn shape(&mut self, raw: &[u8]) -> Shaped { + let seq = self.next; + self.next += 1; + shape(raw, &self.cfg, &self.spill_dir, seq) + } + + /// Whether `text` (a command or a path) refers to a spill file. + pub fn mentions_spill(&self, text: &str) -> bool { + text.contains(&*self.spill_dir.to_string_lossy()) + } +} + +/// Rough token estimate: about four bytes of text per token. +pub fn estimate_tokens(text: &str) -> usize { + text.len().div_ceil(4) +} + +/// Shapes `raw` for the model, saving it to `{spill_dir}/{seq}.log` first. +pub fn shape(raw: &[u8], cfg: &ShapeCfg, spill_dir: &Path, seq: u64) -> Shaped { + let spill_path = spill(raw, spill_dir, seq); + let see_log = |what: &str| match &spill_path { + Some(p) => format!("{what} — full output: {}", p.display()), + None => what.to_string(), + }; + + if is_binary(raw) { + return Shaped { + text: format!( + "[{}]", + see_log(&format!("binary output, {} bytes", raw.len())) + ), + spill_path, + total_lines: 0, + hidden_lines: 0, + collapsed_runs: 0, + binary: true, + }; + } + + let cleaned = strip_ansi(&String::from_utf8_lossy(raw)); + let lines: Vec<&str> = cleaned.lines().map(last_frame).collect(); + let total_lines = lines.len(); + + let (lines, collapsed_hidden, collapsed_runs) = collapse(lines, cfg.collapse_min_run as usize); + let (mut text, cut_lines) = cap(&lines, cfg, &see_log); + if raw.ends_with(b"\n") && !text.is_empty() { + text.push('\n'); + } + Shaped { + text, + spill_path, + total_lines, + hidden_lines: collapsed_hidden + cut_lines, + collapsed_runs, + binary: false, + } +} + +fn spill(raw: &[u8], dir: &Path, seq: u64) -> Option { + let path = dir.join(format!("{seq}.log")); + std::fs::create_dir_all(dir).ok()?; + std::fs::write(&path, raw).ok()?; + Some(path) +} + +/// More than 10% control bytes (other than whitespace and escapes) or any NUL. +fn is_binary(raw: &[u8]) -> bool { + if raw.is_empty() { + return false; + } + let sample = &raw[..raw.len().min(64 * 1024)]; + if sample.contains(&0) { + return true; + } + let control = sample + .iter() + .filter(|&&b| (b < 0x20 && !matches!(b, b'\n' | b'\r' | b'\t' | 0x1b | 0x08)) || b == 0x7f) + .count(); + let invalid = String::from_utf8_lossy(sample) + .chars() + .filter(|&c| c == char::REPLACEMENT_CHARACTER) + .count(); + (control + invalid) * 10 > sample.len() +} + +/// Removes ANSI escape sequences (CSI, OSC and two-byte escapes) and applies +/// backspaces. +pub fn strip_ansi(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\x1b' => match chars.next() { + Some('[') => { + // CSI: parameters, then one final byte in @..~ + for c in chars.by_ref() { + if ('@'..='~').contains(&c) { + break; + } + } + } + Some(']') => { + // OSC: ends with BEL or ESC \ + while let Some(c) = chars.next() { + if c == '\x07' { + break; + } + if c == '\x1b' && chars.peek() == Some(&'\\') { + chars.next(); + break; + } + } + } + // Character-set selection such as ESC ( B takes one more byte. + Some('(' | ')' | '*' | '+' | '-' | '.' | '/' | '#' | '%') => { + chars.next(); + } + _ => {} + }, + '\x08' => { + out.pop(); + } + c if c.is_control() && !matches!(c, '\n' | '\r' | '\t') => {} + c => out.push(c), + } + } + out +} + +/// A line redrawn with carriage returns (progress bars) keeps its last frame. +fn last_frame(line: &str) -> &str { + let line = line.strip_suffix('\r').unwrap_or(line); + match line.rfind('\r') { + Some(i) => &line[i + 1..], + None => line, + } +} + +/// Each run of digits becomes one '#', so "step 9/10" and "step 10/10" look alike. +fn normalized(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut in_digits = false; + for c in line.chars() { + if c.is_ascii_digit() { + if !in_digits { + out.push('#'); + } + in_digits = true; + } else { + out.push(c); + in_digits = false; + } + } + out +} + +/// Replaces each run of `min_run` or more similar lines with its first line, +/// a note and its last line. Returns the lines, how many were hidden and how +/// many runs were collapsed. +fn collapse(lines: Vec<&str>, min_run: usize) -> (Vec, usize, usize) { + if min_run < 3 { + return (lines.into_iter().map(str::to_string).collect(), 0, 0); + } + let mut out = Vec::with_capacity(lines.len()); + let mut hidden = 0; + let mut runs = 0; + let mut i = 0; + while i < lines.len() { + let key = normalized(lines[i]); + let mut j = i + 1; + while j < lines.len() && normalized(lines[j]) == key { + j += 1; + } + let run = j - i; + if run >= min_run && !lines[i].trim().is_empty() { + out.push(lines[i].to_string()); + out.push(format!("[{} similar lines hidden]", run - 2)); + out.push(lines[j - 1].to_string()); + hidden += run - 2; + runs += 1; + } else { + out.extend(lines[i..j].iter().map(|l| l.to_string())); + } + i = j; + } + (out, hidden, runs) +} + +/// Keeps a head and a tail within the token budget. Returns the text and the +/// number of lines cut. +fn cap(lines: &[String], cfg: &ShapeCfg, see_log: &dyn Fn(&str) -> String) -> (String, usize) { + let joined = lines.join("\n"); + let budget = (cfg.cap_tokens as usize).max(64); + if estimate_tokens(&joined) <= budget { + return (joined, 0); + } + let head_budget = (budget as f32 * cfg.head_ratio.clamp(0.0, 1.0)) as usize; + let tail_budget = budget - head_budget; + + let mut head_end = 0; + let mut used = 0; + while head_end < lines.len() { + let t = estimate_tokens(&lines[head_end]) + 1; + if used + t > head_budget { + break; + } + used += t; + head_end += 1; + } + let mut tail_start = lines.len(); + used = 0; + while tail_start > head_end { + let t = estimate_tokens(&lines[tail_start - 1]) + 1; + if used + t > tail_budget { + break; + } + used += t; + tail_start -= 1; + } + + if head_end == 0 && tail_start == lines.len() { + // Not even one whole line fits: cut characters instead. + return (cut_middle(&joined, budget * 4, cfg.head_ratio, see_log), 0); + } + let cut = tail_start - head_end; + let mut parts: Vec = lines[..head_end].to_vec(); + if cut > 0 { + parts.push(format!("[{}]", see_log(&format!("{cut} lines hidden")))); + } + parts.extend_from_slice(&lines[tail_start..]); + let mut text = parts.join("\n"); + + // A few huge lines (minified files, long JSON) can still exceed the budget. + let max_bytes = budget * 4; + if text.len() > max_bytes { + text = cut_middle(&text, max_bytes, cfg.head_ratio, see_log); + } + (text, cut) +} + +fn cut_middle( + text: &str, + max_bytes: usize, + head_ratio: f32, + see_log: &dyn Fn(&str) -> String, +) -> String { + let head = floor_char(text, (max_bytes as f32 * head_ratio) as usize); + let tail = ceil_char(text, text.len() - (max_bytes - head)); + format!( + "{}\n[{}]\n{}", + &text[..head], + see_log(&format!( + "{} characters hidden", + text[head..tail].chars().count() + )), + &text[tail..] + ) +} + +fn floor_char(s: &str, mut i: usize) -> usize { + while !s.is_char_boundary(i) { + i -= 1; + } + i +} + +fn ceil_char(s: &str, mut i: usize) -> usize { + while !s.is_char_boundary(i) { + i += 1; + } + i +} + +#[cfg(test)] +mod tests; diff --git a/crates/shaper/src/tests.rs b/crates/shaper/src/tests.rs new file mode 100644 index 00000000..917fe605 --- /dev/null +++ b/crates/shaper/src/tests.rs @@ -0,0 +1,139 @@ +use super::*; + +fn cfg() -> ShapeCfg { + ShapeCfg { + cap_tokens: 8000, + head_ratio: 0.3, + collapse_min_run: 3, + } +} + +fn run(raw: &[u8], cfg: &ShapeCfg) -> (tempfile::TempDir, Shaped) { + let dir = tempfile::tempdir().unwrap(); + let shaped = shape(raw, cfg, &dir.path().join("out"), 7); + (dir, shaped) +} + +#[test] +fn short_output_passes_through_and_is_spilled() { + let (_d, s) = run(b"hello\nworld\n", &cfg()); + assert_eq!(s.text, "hello\nworld\n"); + assert_eq!((s.total_lines, s.hidden_lines, s.collapsed_runs), (2, 0, 0)); + let path = s.spill_path.unwrap(); + assert!(path.ends_with("out/7.log")); + assert_eq!(std::fs::read(path).unwrap(), b"hello\nworld\n"); +} + +#[test] +fn strips_ansi_sequences_and_backspaces() { + let raw = "\x1b[1;32mPASS\x1b[0m test \x1b]0;title\x07done\x1b(B\nab\x08c\n"; + let (_d, s) = run(raw.as_bytes(), &cfg()); + assert_eq!(s.text, "PASS test done\nac\n"); +} + +#[test] +fn progress_bars_keep_their_last_frame_and_crlf_is_kept_intact() { + let raw = b"Downloading 10%\rDownloading 55%\rDownloading 100%\nline with crlf\r\n"; + let (_d, s) = run(raw, &cfg()); + assert_eq!(s.text, "Downloading 100%\nline with crlf\n"); +} + +#[test] +fn collapses_runs_of_similar_lines() { + let mut raw = String::from("start\n"); + for i in 0..50 { + raw.push_str(&format!("Compiling crate-{i} v0.{i}.0\n")); + } + raw.push_str("Finished\n"); + let (_d, s) = run(raw.as_bytes(), &cfg()); + assert_eq!( + s.text, + "start\nCompiling crate-0 v0.0.0\n[48 similar lines hidden]\nCompiling crate-49 v0.49.0\nFinished\n" + ); + assert_eq!( + (s.hidden_lines, s.collapsed_runs, s.total_lines), + (48, 1, 52) + ); +} + +#[test] +fn short_runs_and_disabled_collapsing_are_left_alone() { + let (_d, s) = run(b"a1\na2\nb\n", &cfg()); + assert_eq!(s.text, "a1\na2\nb\n"); + let mut off = cfg(); + off.collapse_min_run = 0; + let raw: String = (0..10).map(|i| format!("x{i}\n")).collect(); + let (_d, s) = run(raw.as_bytes(), &off); + assert_eq!(s.text, raw); +} + +#[test] +fn caps_long_output_keeping_head_and_tail() { + let mut c = cfg(); + c.cap_tokens = 200; + c.collapse_min_run = 0; + let raw: String = (0..5000).map(|i| format!("line {i} of output\n")).collect(); + let (_d, s) = run(raw.as_bytes(), &c); + assert!(s.text.starts_with("line 0 of output\n")); + assert!(s.text.ends_with("line 4999 of output\n")); + assert!( + s.text.contains("lines hidden — full output: "), + "{}", + s.text + ); + assert!(estimate_tokens(&s.text) < 300); + assert!(s.hidden_lines > 4800); + // About 30% of the kept lines come from the head. + let shown: Vec<&str> = s.text.lines().collect(); + let marker = shown + .iter() + .position(|l| l.contains("lines hidden")) + .unwrap(); + let tail = shown.len() - marker - 1; + assert!(marker * 2 < tail, "head {marker} tail {tail}"); +} + +#[test] +fn caps_a_single_huge_line() { + let mut c = cfg(); + c.cap_tokens = 100; + let raw = format!("{}\n", "é".repeat(10_000)); + let (_d, s) = run(raw.as_bytes(), &c); + assert!(s.text.contains("characters hidden")); + assert!(s.text.len() < 600); +} + +#[test] +fn binary_output_is_described_not_shown() { + let raw: Vec = (0..=255u8).cycle().take(4096).collect(); + let (_d, s) = run(&raw, &cfg()); + assert!(s.binary); + assert!(s + .text + .starts_with("[binary output, 4096 bytes — full output: ")); + assert_eq!(std::fs::read(s.spill_path.unwrap()).unwrap(), raw); + // Mostly-text output with a few odd bytes is still text. + let (_d, s) = run("ok café\n".as_bytes(), &cfg()); + assert!(!s.binary); +} + +#[test] +fn empty_output_stays_empty() { + let (_d, s) = run(b"", &cfg()); + assert_eq!(s.text, ""); + assert_eq!(s.total_lines, 0); +} + +#[test] +fn shaper_numbers_spill_files_and_spots_spill_reads() { + let dir = tempfile::tempdir().unwrap(); + let spill_dir = dir.path().join("out"); + let mut shaper = Shaper::new(cfg(), &spill_dir); + let a = shaper.shape(b"one\n"); + let b = shaper.shape(b"two\n"); + assert!(a.spill_path.unwrap().ends_with("out/1.log")); + assert!(b.spill_path.unwrap().ends_with("out/2.log")); + let cmd = format!("grep error {}/2.log", spill_dir.display()); + assert!(shaper.mentions_spill(&cmd)); + assert!(!shaper.mentions_spill("grep error build.log")); +} diff --git a/crates/terminal/Cargo.toml b/crates/terminal/Cargo.toml new file mode 100644 index 00000000..f8e26876 --- /dev/null +++ b/crates/terminal/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "gitagent-terminal" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +portable-pty = "0.9.0" +tokio.workspace = true + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/terminal/src/buffer.rs b/crates/terminal/src/buffer.rs new file mode 100644 index 00000000..a9aecf8c --- /dev/null +++ b/crates/terminal/src/buffer.rs @@ -0,0 +1,202 @@ +//! Output buffer for one shell, with end-marker detection and a memory cap. + +/// The parsed end-marker line `__GA_END_{seq}__ {exit} {cwd}`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Marker { + pub exit: i32, + pub cwd: String, +} + +pub(crate) struct Taken { + pub output: Vec, + pub omitted_bytes: u64, +} + +pub(crate) fn marker_prefix(seq: u64) -> String { + format!("__GA_END_{seq}__ ") +} + +/// Raw bytes read from the PTY since the last result was taken. +/// +/// When the buffer grows past `cap` bytes the middle is dropped, keeping the +/// first `cap / 4` and the last `3 * cap / 4` bytes and counting what was +/// dropped. The marker always arrives last, so dropping the middle never loses it. +pub(crate) struct OutputBuffer { + data: Vec, + cap: usize, + /// Bytes before this index contain no marker start for the current command. + scanned: usize, + omitted: u64, +} + +impl OutputBuffer { + pub fn new(cap: usize) -> Self { + OutputBuffer { + data: Vec::new(), + cap: cap.max(1024), + scanned: 0, + omitted: 0, + } + } + + fn head_len(&self) -> usize { + self.cap / 4 + } + + pub fn push(&mut self, bytes: &[u8]) { + self.data.extend_from_slice(bytes); + if self.data.len() <= self.cap { + return; + } + let head = self.head_len(); + let drop_end = self.data.len() - (self.cap - head); + let dropped = drop_end - head; + self.data.drain(head..drop_end); + self.omitted += dropped as u64; + self.scanned = if self.scanned >= drop_end { + self.scanned - dropped + } else { + self.scanned.min(head) + }; + } + + /// Looks for the complete marker line for `seq`. On success, returns the + /// marker and the output before it, leaving anything after the marker line + /// in the buffer. + pub fn take_marker(&mut self, seq: u64) -> Option<(Marker, Taken)> { + let needle = format!("\n{}", marker_prefix(seq)).into_bytes(); + let mut from = self.scanned; + loop { + let Some(rel) = find(&self.data[from..], &needle) else { + // Leave room for a marker that is only partly received. + self.scanned = self.data.len().saturating_sub(needle.len()).max(from); + return None; + }; + let start = from + rel; + let line_start = start + needle.len(); + let Some(nl) = self.data[line_start..].iter().position(|&b| b == b'\n') else { + // Marker line not complete yet. + self.scanned = start; + return None; + }; + let line = String::from_utf8_lossy(&self.data[line_start..line_start + nl]); + let (code, cwd) = line.split_once(' ').unwrap_or((&line, "")); + let Ok(exit) = code.parse::() else { + // Not our marker (for example the text of the printf format). + from = line_start; + continue; + }; + let marker = Marker { + exit, + cwd: cwd.to_string(), + }; + let rest = self.data.split_off(line_start + nl + 1); + self.data.truncate(start); + let taken = self.take_upto(self.data.len()); + self.data = rest; + return Some((marker, taken)); + } + } + + /// Takes the output received so far for a command that is still running, + /// holding back any bytes that might be the start of its marker line. + pub fn take_partial(&mut self, seq: u64) -> Taken { + let needle = format!("\n{}", marker_prefix(seq)).into_bytes(); + let keep_from = match find(&self.data[self.scanned.min(self.data.len())..], &needle) { + Some(rel) => self.scanned + rel, + None => { + let longest = (1..needle.len()) + .rev() + .find(|&k| self.data.ends_with(&needle[..k])) + .unwrap_or(0); + self.data.len() - longest + } + }; + self.take_upto(keep_from) + } + + /// Takes everything buffered so far. + pub fn take_all(&mut self) -> Taken { + self.take_upto(self.data.len()) + } + + fn take_upto(&mut self, end: usize) -> Taken { + let rest = self.data.split_off(end); + let mut output = std::mem::replace(&mut self.data, rest); + if self.omitted > 0 { + let note = format!("\n[... {} bytes omitted ...]\n", self.omitted); + let at = self.head_len().min(output.len()); + output.splice(at..at, note.into_bytes()); + } + let taken = Taken { + output, + omitted_bytes: self.omitted, + }; + self.scanned = 0; + self.omitted = 0; + taken + } +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn finds_marker_split_across_pushes() { + let mut b = OutputBuffer::new(1 << 20); + b.push(b"hello\nworld\n__GA_EN"); + assert!(b.take_marker(3).is_none()); + b.push(b"D_3__ 42 /tmp/a b"); + assert!(b.take_marker(3).is_none()); + b.push(b"\nleftover"); + let (m, t) = b.take_marker(3).unwrap(); + assert_eq!(m.exit, 42); + assert_eq!(m.cwd, "/tmp/a b"); + assert_eq!(t.output, b"hello\nworld"); + assert_eq!(b.take_all().output, b"leftover"); + } + + #[test] + fn ignores_markers_for_other_commands_and_format_strings() { + let mut b = OutputBuffer::new(1 << 20); + b.push(b"x\n__GA_END_2__ 0 /\n\n__GA_END_3__ %s %s\ny\n__GA_END_3__ 1 /\n"); + let (m, t) = b.take_marker(3).unwrap(); + assert_eq!(m.exit, 1); + assert_eq!(t.output, b"x\n__GA_END_2__ 0 /\n\n__GA_END_3__ %s %s\ny"); + } + + #[test] + fn partial_take_holds_back_a_split_marker() { + let mut b = OutputBuffer::new(1 << 20); + b.push(b"progress 50%\n__GA_E"); + assert!(b.take_marker(9).is_none()); + assert_eq!(b.take_partial(9).output, b"progress 50%"); + b.push(b"ND_9__ 0 /\n"); + let (m, t) = b.take_marker(9).unwrap(); + assert_eq!(m.exit, 0); + assert!(t.output.is_empty()); + } + + #[test] + fn caps_memory_keeping_head_and_tail() { + let mut b = OutputBuffer::new(4096); + b.push(b"HEAD"); + for _ in 0..100 { + b.push(&[b'x'; 1000]); + assert!(b.take_marker(1).is_none()); + } + b.push(b"TAIL\n__GA_END_1__ 0 /\n"); + let (_, t) = b.take_marker(1).unwrap(); + assert!(t.omitted_bytes > 90_000); + let text = String::from_utf8(t.output).unwrap(); + assert!(text.starts_with("HEAD")); + assert!(text.ends_with("TAIL")); + assert!(text.contains("bytes omitted")); + assert!(text.len() < 5000); + } +} diff --git a/crates/terminal/src/keys.rs b/crates/terminal/src/keys.rs new file mode 100644 index 00000000..558ede43 --- /dev/null +++ b/crates/terminal/src/keys.rs @@ -0,0 +1,70 @@ +//! Turns the text a model sends to `write_stdin` into bytes for the terminal. +//! +//! Models usually write control keys as escape sequences in plain text +//! (`\x03` for Ctrl-C), so those are decoded here. Real control characters +//! that arrive already decoded pass through unchanged. + +/// Decodes `\xHH`, `\e`, `\n`, `\r`, `\t`, `\0` and `\\`. Anything else after a +/// backslash is kept as written. +pub fn decode_keys(s: &str) -> Vec { + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != b'\\' || i + 1 >= bytes.len() { + out.push(bytes[i]); + i += 1; + continue; + } + let next = bytes[i + 1]; + let simple = match next { + b'e' => Some(0x1b), + b'n' => Some(b'\n'), + b'r' => Some(b'\r'), + b't' => Some(b'\t'), + b'0' => Some(0), + b'\\' => Some(b'\\'), + _ => None, + }; + if let Some(b) = simple { + out.push(b); + i += 2; + } else if next == b'x' && hex_pair(&bytes[i + 2..]).is_some() { + out.push(hex_pair(&bytes[i + 2..]).unwrap_or(0)); + i += 4; + } else { + out.push(b'\\'); + i += 1; + } + } + out +} + +fn hex_pair(b: &[u8]) -> Option { + let hex = std::str::from_utf8(b.get(..2)?).ok()?; + u8::from_str_radix(hex, 16).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_control_keys() { + assert_eq!(decode_keys(r"\x03"), [3]); + assert_eq!(decode_keys(r"\x1b[A"), b"\x1b[A"); + assert_eq!(decode_keys(r"\e:wq\r"), b"\x1b:wq\r"); + assert_eq!(decode_keys(r"print(1)\n"), b"print(1)\n"); + assert_eq!(decode_keys("\u{3}"), [3]); + assert_eq!(decode_keys(r"a\\nb"), br"a\nb"); + } + + #[test] + fn leaves_unknown_and_incomplete_escapes_alone() { + assert_eq!(decode_keys(r"C:\path"), br"C:\path"); + assert_eq!(decode_keys(r"\xZZ"), br"\xZZ"); + assert_eq!(decode_keys(r"\x4"), br"\x4"); + assert_eq!(decode_keys("end\\"), b"end\\"); + assert_eq!(decode_keys(""), b""); + } +} diff --git a/crates/terminal/src/lib.rs b/crates/terminal/src/lib.rs new file mode 100644 index 00000000..0e149aaf --- /dev/null +++ b/crates/terminal/src/lib.rs @@ -0,0 +1,524 @@ +//! Persistent shell sessions on a PTY. +//! +//! Each command is written to a script file and run with one short input line, +//! `. 'script'; printf '\n__GA_END_{seq}__ %s %s\n' "$?" "$PWD"`, so that +//! quotes, heredocs and commands reading stdin cannot swallow the end marker. +//! The shell runs interactively (for job control, so Ctrl-C reaches only the +//! foreground job) with line editing, echo and prompts turned off. + +mod buffer; +mod keys; +mod manager; + +pub use keys::decode_keys; +pub use manager::{Terminal, MAIN_SESSION, MAX_SESSIONS}; + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use portable_pty::{native_pty_system, Child, CommandBuilder, MasterPty, PtySize}; +use tokio::sync::mpsc; +use tokio::time::timeout_at; + +use buffer::{marker_prefix, OutputBuffer, Taken}; + +/// Environment set for every shell: no pagers, colors or interactive prompts. +const SHELL_ENV: &[(&str, &str)] = &[ + ("TERM", "dumb"), + ("NO_COLOR", "1"), + ("PAGER", "cat"), + ("GIT_PAGER", "cat"), + ("MANPAGER", "cat"), + ("SYSTEMD_PAGER", ""), + ("GIT_TERMINAL_PROMPT", "0"), + ("DEBIAN_FRONTEND", "noninteractive"), + ("PIP_PROGRESS_BAR", "off"), + ("PYTHONUNBUFFERED", "1"), + ("HISTFILE", "/dev/null"), +]; + +/// Run once after the shell starts. Echo, CR translation and prompts would +/// otherwise end up in command output. +const INIT_LINE: &str = + "stty -echo -onlcr 2>/dev/null; PS1=''; PS2=''; PS0=''; PROMPT_COMMAND=''; set +o history 2>/dev/null"; + +const READ_CHUNK: usize = 8192; +const STARTUP_TIMEOUT: Duration = Duration::from_secs(15); +const INTERRUPT_WAIT: Duration = Duration::from_secs(3); + +#[derive(Debug, Clone)] +pub struct ShellConfig { + /// Initial working directory. + pub cwd: PathBuf, + /// Where command scripts are written; created if missing. + pub script_dir: PathBuf, + /// Variables removed from the inherited environment (API keys, etc.). + pub remove_env: Vec, + /// Extra variables, applied after `SHELL_ENV`. + pub env: Vec<(String, String)>, + /// Extra commands run once when the shell starts (e.g. restoring the + /// environment of another session). + pub init: Vec, + /// Program and arguments that start the shell for us (the sandbox + /// launcher); the shell command follows a `--`. + pub launcher: Vec, + /// In-memory output cap per command, in bytes. + pub max_buffer_bytes: usize, + pub cols: u16, + pub rows: u16, +} + +impl ShellConfig { + pub fn new(cwd: impl Into, script_dir: impl Into) -> Self { + ShellConfig { + cwd: cwd.into(), + script_dir: script_dir.into(), + remove_env: Vec::new(), + env: Vec::new(), + init: Vec::new(), + launcher: Vec::new(), + max_buffer_bytes: 1 << 20, + cols: 200, + rows: 50, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExecResult { + /// Session that ran the command (set by [`Terminal`]; 0 for a bare [`Session`]). + pub session: u32, + pub output: Vec, + /// Exit code, when the command finished and the shell survived. + pub exit: Option, + /// False when the wait time ran out first; the command keeps running. + pub finished: bool, + /// Bytes dropped from the middle of very large output. + pub omitted_bytes: u64, + pub elapsed: Duration, + /// A new shell had to be started for this command. + pub shell_restarted: bool, + /// Things the model should know, e.g. that the shell was restarted. + pub notes: Vec, +} + +enum ReadEvent { + Data(Vec), + Eof, +} + +struct Shell { + master: Box, + writer: Box, + child: Box, + rx: mpsc::UnboundedReceiver, + buf: OutputBuffer, + eof: bool, +} + +/// One persistent shell. If the shell exits (e.g. `exit` or `set -e`), a new +/// one is started in the last known directory on the next command. +pub struct Session { + cfg: ShellConfig, + shell: Option, + seq: u64, + /// Sequence number of a command still running after its wait time ran out. + running: Option, + /// Set when Ctrl-C was sent to the running command. + interrupted: bool, + cwd: PathBuf, +} + +impl Session { + pub async fn start(cfg: ShellConfig) -> Result { + std::fs::create_dir_all(&cfg.script_dir) + .with_context(|| format!("creating {}", cfg.script_dir.display()))?; + let mut session = Session { + cwd: cfg.cwd.clone(), + cfg, + shell: None, + seq: 0, + running: None, + interrupted: false, + }; + // Starting a shell very rarely fails right after the fork (seen under + // heavy parallel load); one fresh attempt is enough. + if session.spawn().await.is_err() { + session.kill(); + session.spawn().await?; + } + Ok(session) + } + + /// Directory the shell was in after the last finished command. + pub fn cwd(&self) -> &Path { + &self.cwd + } + + pub fn is_running(&self) -> bool { + self.running.is_some() + } + + /// False after the shell exited; the next `exec` starts a new one. + pub fn is_alive(&self) -> bool { + self.shell.is_some() + } + + /// Runs `cmd`, waiting up to `yield_ms` for it to finish. A command still + /// running from an earlier call is interrupted first. + pub async fn exec(&mut self, cmd: &str, yield_ms: u64) -> Result { + let started = Instant::now(); + let mut notes = Vec::new(); + if self.running.is_some() { + notes.extend(self.interrupt_running().await?); + } + let mut restarted = false; + if self.shell.is_none() { + self.spawn().await?; + restarted = true; + notes.push(format!( + "A new shell was started in {} (shell variables, functions and background jobs from before were lost).", + self.cwd.display() + )); + } + + self.seq += 1; + let seq = self.seq; + let script = self.cfg.script_dir.join(format!("cmd-{seq}.sh")); + let mut body = cmd.to_string(); + if !body.ends_with('\n') { + body.push('\n'); + } + std::fs::write(&script, body).with_context(|| format!("writing {}", script.display()))?; + let line = format!( + ". {}; {}\n", + shell_quote(&script), + marker_command(seq, "\"$?\"") + ); + self.write_input(line.as_bytes())?; + self.running = Some(seq); + self.interrupted = false; + + let mut result = self.wait(seq, Duration::from_millis(yield_ms)).await?; + result.elapsed = started.elapsed(); + result.shell_restarted = restarted; + notes.append(&mut result.notes); + result.notes = notes; + Ok(result) + } + + /// Sends `keys` to the running command and waits up to `yield_ms` for it + /// to finish. Empty `keys` just waits. After Ctrl-C (`0x03`), if the shell + /// itself took the interrupt and dropped the rest of its input line, it is + /// asked to print the end marker again. + pub async fn write_stdin(&mut self, keys: &[u8], yield_ms: u64) -> Result { + let started = Instant::now(); + let Some(seq) = self.running else { + bail!("no command is running"); + }; + if !keys.is_empty() { + self.write_input(keys)?; + } + if keys.contains(&0x03) { + self.interrupted = true; + } + let mut result = self.wait(seq, Duration::from_millis(yield_ms)).await?; + if !result.finished && self.interrupted && self.shell_in_foreground() { + self.interrupted = false; + let line = format!("{}\n", marker_command(seq, "130")); + self.write_input(line.as_bytes())?; + let more = self.wait(seq, INTERRUPT_WAIT).await?; + result.output.extend_from_slice(&more.output); + result.omitted_bytes += more.omitted_bytes; + result.exit = more.exit; + result.finished = more.finished; + result.notes.extend(more.notes); + } + result.elapsed = started.elapsed(); + Ok(result) + } + + /// True when the shell, not a program it started, owns the terminal: it is + /// at its prompt or running a shell builtin. + fn shell_in_foreground(&self) -> bool { + let Some(shell) = &self.shell else { + return false; + }; + match ( + shell.master.process_group_leader(), + shell.child.process_id(), + ) { + (Some(pgrp), Some(pid)) => i64::from(pgrp) == i64::from(pid), + _ => false, + } + } + + /// Waits up to `yield_ms` more for a command that is still running. + pub async fn poll(&mut self, yield_ms: u64) -> Result { + let started = Instant::now(); + let Some(seq) = self.running else { + bail!("no command is running"); + }; + let mut result = self.wait(seq, Duration::from_millis(yield_ms)).await?; + result.elapsed = started.elapsed(); + Ok(result) + } + + /// Reads and discards output for up to `max_wait`, so programs writing to + /// the terminal never block. Notices when the shell exits. + pub async fn drain(&mut self, max_wait: Duration) { + let deadline = tokio::time::Instant::now() + max_wait; + let Some(shell) = self.shell.as_mut() else { + return; + }; + let exited = loop { + match timeout_at(deadline, shell.rx.recv()).await { + Ok(Some(ReadEvent::Data(_))) => {} + Ok(Some(ReadEvent::Eof)) | Ok(None) => break true, + Err(_) => break false, + } + }; + if exited { + let _ = shell.child.wait(); + self.shell = None; + self.running = None; + } + } + + /// Stops the shell and everything started from it. + pub fn kill(&mut self) { + if let Some(mut shell) = self.shell.take() { + let _ = shell.child.kill(); + let _ = shell.child.wait(); + } + self.running = None; + } + + async fn spawn(&mut self) -> Result<()> { + let pair = native_pty_system() + .openpty(PtySize { + rows: self.cfg.rows, + cols: self.cfg.cols, + pixel_width: 0, + pixel_height: 0, + }) + .context("opening a pty")?; + + let (shell, shell_args): (String, &[&str]) = match find_in_path("bash") { + Some(bash) => ( + bash.to_string_lossy().into_owned(), + &["--noprofile", "--norc", "--noediting", "-i"], + ), + None => ("sh".to_string(), &["-i"]), + }; + let mut cmd = match self.cfg.launcher.split_first() { + Some((program, args)) => { + let mut c = CommandBuilder::new(program); + c.args(args); + c.arg("--"); + c.arg(&shell); + c + } + None => CommandBuilder::new(&shell), + }; + cmd.args(shell_args); + let cwd = if self.cwd.is_dir() { + self.cwd.clone() + } else { + self.cfg.cwd.clone() + }; + cmd.cwd(&cwd); + for key in &self.cfg.remove_env { + cmd.env_remove(key); + } + for (k, v) in SHELL_ENV { + cmd.env(k, v); + } + for (k, v) in &self.cfg.env { + cmd.env(k, v); + } + + let child = pair + .slave + .spawn_command(cmd) + .context("starting the shell")?; + drop(pair.slave); + let reader = pair.master.try_clone_reader().context("pty reader")?; + let writer = pair.master.take_writer().context("pty writer")?; + let (tx, rx) = mpsc::unbounded_channel(); + std::thread::spawn(move || read_loop(reader, tx)); + + self.shell = Some(Shell { + master: pair.master, + writer, + child, + rx, + buf: OutputBuffer::new(self.cfg.max_buffer_bytes), + eof: false, + }); + self.cwd = cwd; + self.running = None; + + // Initialize, then discard everything up to the first marker (seq 0 + // is never used for commands). + let mut init = INIT_LINE.to_string(); + for extra in &self.cfg.init { + init.push_str("; "); + init.push_str(extra); + } + let line = format!("{init}; {}\n", marker_command(0, "0")); + let mut attempt = 0; + while let Err(e) = self.write_input(line.as_bytes()) { + attempt += 1; + if attempt >= 10 { + return Err(e); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let res = self.wait(0, STARTUP_TIMEOUT).await?; + if !res.finished || res.exit.is_none() { + self.kill(); + bail!( + "the shell did not start: {}", + String::from_utf8_lossy(&res.output) + ); + } + Ok(()) + } + + /// Sends Ctrl-C to the running command and waits for it to stop. If the + /// shell itself was interrupted (no marker comes back), asks it to print + /// the marker again; if that fails too, restarts the shell. + async fn interrupt_running(&mut self) -> Result> { + let Some(seq) = self.running else { + return Ok(Vec::new()); + }; + self.write_input(b"\x03")?; + let res = self.wait(seq, INTERRUPT_WAIT).await?; + if res.finished { + return Ok(vec![ + "The previous command was still running and was interrupted with Ctrl-C.".into(), + ]); + } + // Only re-ask for the marker when the shell is back in charge; otherwise + // the line would be typed into the still-running program. + if self.shell_in_foreground() { + let line = format!("{}\n", marker_command(seq, "130")); + self.write_input(line.as_bytes())?; + if self.wait(seq, INTERRUPT_WAIT).await?.finished { + return Ok(vec![ + "The previous command was still running and was interrupted with Ctrl-C." + .into(), + ]); + } + } + self.kill(); + Ok(vec![ + "The previous command did not stop on Ctrl-C, so its shell was killed.".into(), + ]) + } + + fn write_input(&mut self, bytes: &[u8]) -> Result<()> { + let shell = self.shell.as_mut().context("no shell")?; + shell + .writer + .write_all(bytes) + .context("writing to the shell")?; + shell.writer.flush().context("writing to the shell")?; + Ok(()) + } + + async fn wait(&mut self, seq: u64, max_wait: Duration) -> Result { + let deadline = tokio::time::Instant::now() + max_wait; + let shell = self.shell.as_mut().context("no shell")?; + loop { + if let Some((marker, taken)) = shell.buf.take_marker(seq) { + self.running = None; + if !marker.cwd.is_empty() { + self.cwd = PathBuf::from(&marker.cwd); + } + return Ok(result(taken, Some(marker.exit), true, Vec::new())); + } + if shell.eof { + let taken = shell.buf.take_all(); + let status = shell.child.wait().ok(); + self.shell = None; + self.running = None; + let code = status.map_or("unknown".to_string(), |s| s.exit_code().to_string()); + let note = format!( + "The shell exited (status {code}), e.g. because of `exit` or `set -e`. The next command starts a new shell in {}.", + self.cwd.display() + ); + return Ok(result(taken, None, true, vec![note])); + } + match timeout_at(deadline, shell.rx.recv()).await { + Ok(Some(ReadEvent::Data(bytes))) => shell.buf.push(&bytes), + Ok(Some(ReadEvent::Eof)) | Ok(None) => shell.eof = true, + Err(_) => { + let taken = shell.buf.take_partial(seq); + return Ok(result(taken, None, false, Vec::new())); + } + } + } + } +} + +impl Drop for Session { + fn drop(&mut self) { + self.kill(); + } +} + +fn result(taken: Taken, exit: Option, finished: bool, notes: Vec) -> ExecResult { + ExecResult { + session: 0, + output: taken.output, + exit, + finished, + omitted_bytes: taken.omitted_bytes, + elapsed: Duration::ZERO, + shell_restarted: false, + notes, + } +} + +fn marker_command(seq: u64, status: &str) -> String { + format!( + "printf '\\n{}%s %s\\n' {status} \"$PWD\"", + marker_prefix(seq) + ) +} + +fn read_loop(mut reader: Box, tx: mpsc::UnboundedSender) { + let mut chunk = vec![0u8; READ_CHUNK]; + loop { + match reader.read(&mut chunk) { + // Linux reports EIO instead of EOF once the shell side closes. + Ok(0) | Err(_) => { + let _ = tx.send(ReadEvent::Eof); + return; + } + Ok(n) => { + if tx.send(ReadEvent::Data(chunk[..n].to_vec())).is_err() { + return; + } + } + } + } +} + +pub(crate) fn shell_quote(path: &Path) -> String { + format!("'{}'", path.display().to_string().replace('\'', r"'\''")) +} + +fn find_in_path(name: &str) -> Option { + let path = std::env::var_os("PATH")?; + std::env::split_paths(&path) + .map(|dir| dir.join(name)) + .find(|p| p.is_file()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/terminal/src/manager.rs b/crates/terminal/src/manager.rs new file mode 100644 index 00000000..873f8452 --- /dev/null +++ b/crates/terminal/src/manager.rs @@ -0,0 +1,254 @@ +//! The main shell plus background sessions for servers, REPLs and other +//! commands that keep running. + +use std::collections::BTreeMap; +use std::path::Path; +use std::time::Duration; + +use anyhow::{bail, Result}; + +use crate::{shell_quote, ExecResult, Session, ShellConfig}; + +pub const MAIN_SESSION: u32 = 1; +/// Upper bound on sessions alive at once, the main one included. +pub const MAX_SESSIONS: usize = 8; + +pub struct Terminal { + base: ShellConfig, + sessions: BTreeMap, + next_id: u32, +} + +impl Terminal { + pub async fn start(base: ShellConfig) -> Result { + let main = Session::start(session_cfg(&base, MAIN_SESSION)).await?; + Ok(Terminal { + base, + sessions: BTreeMap::from([(MAIN_SESSION, main)]), + next_id: MAIN_SESSION + 1, + }) + } + + fn main(&mut self) -> &mut Session { + self.sessions + .get_mut(&MAIN_SESSION) + .expect("the main session always exists") + } + + /// Working directory of the main session. + pub fn cwd(&self) -> &Path { + self.sessions[&MAIN_SESSION].cwd() + } + + /// Ids of background sessions still running a command. + pub fn background_sessions(&self) -> Vec { + self.sessions + .keys() + .copied() + .filter(|&id| id != MAIN_SESSION) + .collect() + } + + /// Runs `cmd` in the main session, or in a new background session that + /// starts with the main session's exported variables and directory. + pub async fn exec(&mut self, cmd: &str, yield_ms: u64, background: bool) -> Result { + if !background { + let mut r = self.main().exec(cmd, yield_ms).await?; + r.session = MAIN_SESSION; + return Ok(r); + } + if self.sessions.len() >= MAX_SESSIONS { + bail!( + "too many sessions are running ({}); stop one first, e.g. write_stdin with \"\\x03\". Running: {:?}", + self.sessions.len(), + self.background_sessions() + ); + } + let id = self.next_id; + self.next_id += 1; + + let mut cfg = session_cfg(&self.base, id); + let env_file = self.base.script_dir.join(format!("env-{id}.sh")); + let main = self.main(); + if main.is_alive() && !main.is_running() { + let export = format!("export -p > {}", shell_quote(&env_file)); + if main.exec(&export, 5000).await?.exit == Some(0) { + cfg.init + .push(format!(". {} 2>/dev/null", shell_quote(&env_file))); + } + } + cfg.cwd = main.cwd().to_path_buf(); + cfg.init + .push(format!("cd {} 2>/dev/null", shell_quote(&cfg.cwd))); + + let mut session = Session::start(cfg).await?; + let mut r = session.exec(cmd, yield_ms).await?; + r.session = id; + if session.is_running() { + self.sessions.insert(id, session); + } + Ok(r) + } + + /// Sends `keys` to a running session (empty keys just wait for output). + pub async fn write_stdin(&mut self, id: u32, keys: &[u8], yield_ms: u64) -> Result { + let running = self.background_sessions(); + let Some(session) = self.sessions.get_mut(&id) else { + bail!("there is no session {id}; running background sessions: {running:?}"); + }; + if !session.is_running() { + bail!("session {id} is not running a command; use exec to run one"); + } + let mut r = session.write_stdin(keys, yield_ms).await?; + r.session = id; + // A background session is done once its command finishes. + if id != MAIN_SESSION && !session.is_running() { + self.sessions.remove(&id); + } + Ok(r) + } + + /// Keeps every shell open, discarding their output, until all of them + /// exit. Processes the agent left running (servers, background jobs) keep + /// a live terminal, as they would in a terminal multiplexer. + pub async fn linger(&mut self) { + loop { + let mut alive = false; + for session in self.sessions.values_mut() { + if session.is_alive() { + alive = true; + session.drain(Duration::from_millis(500)).await; + } + } + if !alive { + return; + } + } + } + + /// Stops every session. + pub fn kill_all(&mut self) { + for session in self.sessions.values_mut() { + session.kill(); + } + self.sessions.retain(|&id, _| id == MAIN_SESSION); + } +} + +fn session_cfg(base: &ShellConfig, id: u32) -> ShellConfig { + let mut cfg = base.clone(); + cfg.script_dir = base.script_dir.join(format!("session-{id}")); + cfg +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(r: &ExecResult) -> String { + String::from_utf8_lossy(&r.output).into_owned() + } + + async fn start() -> (tempfile::TempDir, Terminal) { + let dir = tempfile::tempdir().unwrap(); + let t = Terminal::start(ShellConfig::new(dir.path(), dir.path().join("scripts"))) + .await + .unwrap(); + (dir, t) + } + + #[tokio::test] + async fn background_session_inherits_env_and_directory() { + let (_d, mut t) = start().await; + t.exec("mkdir -p sub && cd sub && export GA_V=7", 5000, false) + .await + .unwrap(); + let r = t + .exec("echo $GA_V $(basename $PWD); sleep 30", 1000, true) + .await + .unwrap(); + assert_eq!(r.session, 2); + assert!(!r.finished); + // A trailing newline is held back while the command runs (it could + // start the end marker). + assert_eq!(text(&r).trim_end(), "7 sub"); + assert_eq!(t.background_sessions(), vec![2]); + + // The main session is free while the background one runs. + let r = t.exec("echo main", 5000, false).await.unwrap(); + assert_eq!((r.session, text(&r).as_str()), (MAIN_SESSION, "main\n")); + + let r = t.write_stdin(2, b"\x03", 3000).await.unwrap(); + assert!(r.finished); + assert!(t.background_sessions().is_empty()); + } + + #[tokio::test] + async fn repl_in_a_background_session() { + let (_d, mut t) = start().await; + let r = t.exec("cat -n", 300, true).await.unwrap(); + assert!(!r.finished); + let r = t.write_stdin(r.session, b"hello\n", 1000).await.unwrap(); + assert!(text(&r).contains("1\thello"), "{:?}", text(&r)); + // Ctrl-D ends cat; the session finishes and goes away. + let r = t.write_stdin(2, b"\x04", 3000).await.unwrap(); + assert!(r.finished); + assert_eq!(r.exit, Some(0)); + assert!(t.write_stdin(2, b"", 100).await.is_err()); + } + + #[tokio::test] + async fn poll_and_interrupt_the_main_session() { + let (_d, mut t) = start().await; + let r = t.exec("sleep 1; echo slow", 100, false).await.unwrap(); + assert!(!r.finished); + let r = t.write_stdin(MAIN_SESSION, b"", 5000).await.unwrap(); + assert!(r.finished); + assert_eq!(text(&r), "slow\n"); + + // Interrupting a shell builtin loop: the shell takes the Ctrl-C and + // is asked for the end marker again. + t.exec("while true; do :; done", 300, false).await.unwrap(); + let r = t.write_stdin(MAIN_SESSION, b"\x03", 2000).await.unwrap(); + assert!(r.finished, "{r:?}"); + let r = t.exec("echo alive", 5000, false).await.unwrap(); + assert_eq!(text(&r), "alive\n"); + } + + #[tokio::test] + async fn linger_keeps_jobs_running_until_the_shells_exit() { + let (dir, mut t) = start().await; + let marker = dir.path().join("marker"); + let cmd = format!("(sleep 1; echo late; touch {}) &", marker.display()); + t.exec(&cmd, 5000, false).await.unwrap(); + // The shells stay up, so linger only ends with the timeout here... + let res = tokio::time::timeout(Duration::from_secs(3), t.linger()).await; + assert!(res.is_err()); + // ...and the job, which wrote to the terminal, finished meanwhile. + assert!(marker.exists()); + // Once every shell has exited, linger returns. + t.exec("exit", 5000, false).await.unwrap(); + tokio::time::timeout(Duration::from_secs(3), t.linger()) + .await + .expect("linger returns when no shell is left"); + } + + #[tokio::test] + async fn errors_for_unknown_or_idle_sessions_and_session_limit() { + let (_d, mut t) = start().await; + assert!(t.write_stdin(9, b"x", 100).await.is_err()); + assert!(t.write_stdin(MAIN_SESSION, b"x", 100).await.is_err()); + // A background command that finishes at once leaves no session behind. + let r = t.exec("echo quick", 5000, true).await.unwrap(); + assert!(r.finished); + assert!(t.background_sessions().is_empty()); + + for _ in 1..MAX_SESSIONS { + t.exec("sleep 30", 100, true).await.unwrap(); + } + let err = t.exec("sleep 30", 100, true).await.unwrap_err(); + assert!(err.to_string().contains("too many sessions")); + t.kill_all(); + assert!(t.background_sessions().is_empty()); + } +} diff --git a/crates/terminal/src/tests.rs b/crates/terminal/src/tests.rs new file mode 100644 index 00000000..916a8b71 --- /dev/null +++ b/crates/terminal/src/tests.rs @@ -0,0 +1,146 @@ +use super::*; + +struct Fixture { + _dir: tempfile::TempDir, + session: Session, +} + +async fn start() -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let cfg = ShellConfig::new(dir.path(), dir.path().join("scripts")); + let session = Session::start(cfg).await.unwrap(); + Fixture { _dir: dir, session } +} + +fn text(r: &ExecResult) -> String { + String::from_utf8_lossy(&r.output).into_owned() +} + +#[tokio::test] +async fn runs_a_command_and_reports_the_exit_code() { + let mut f = start().await; + let r = f.session.exec("echo hi", 5000).await.unwrap(); + assert_eq!(text(&r), "hi\n"); + assert_eq!(r.exit, Some(0)); + assert!(r.finished); + let r = f.session.exec("echo oops >&2; false", 5000).await.unwrap(); + assert_eq!(text(&r), "oops\n"); + assert_eq!(r.exit, Some(1)); +} + +#[tokio::test] +async fn keeps_state_between_commands() { + let mut f = start().await; + f.session + .exec("mkdir -p sub && cd sub && export GA_X=42", 5000) + .await + .unwrap(); + let r = f + .session + .exec("echo $GA_X; basename $PWD", 5000) + .await + .unwrap(); + assert_eq!(text(&r), "42\nsub\n"); + assert!(f.session.cwd().ends_with("sub")); +} + +#[tokio::test] +async fn handles_quotes_heredocs_and_stdin_readers() { + let mut f = start().await; + let cmd = "cat > f.txt <<'EOF'\nit's \"quoted\" $HOME\nEOF\ncat f.txt"; + let r = f.session.exec(cmd, 5000).await.unwrap(); + assert_eq!(text(&r), "it's \"quoted\" $HOME\n"); + // An unterminated quote is a syntax error, not a hang. + let r = f.session.exec("echo 'unterminated", 5000).await.unwrap(); + assert!(r.finished); + assert_ne!(r.exit, Some(0)); + // A command reading stdin must not consume the marker. + let r = f + .session + .exec("read -t 1 x; echo got:$x", 5000) + .await + .unwrap(); + assert!(r.finished); + assert_eq!(text(&r), "got:\n"); +} + +#[tokio::test] +async fn yields_partial_output_then_interrupts_on_next_exec() { + let mut f = start().await; + let r = f + .session + .exec("echo start; sleep 30; echo never", 1000) + .await + .unwrap(); + assert!(!r.finished); + assert_eq!(r.exit, None); + assert_eq!(text(&r).trim(), "start"); + assert!(f.session.is_running()); + + let r = f.session.exec("echo next", 5000).await.unwrap(); + assert_eq!(text(&r), "next\n"); + assert!(r.notes[0].contains("interrupted"), "{:?}", r.notes); + assert!(!f.session.is_running()); +} + +#[tokio::test] +async fn interrupts_a_shell_builtin_loop() { + let mut f = start().await; + let r = f.session.exec("while true; do :; done", 500).await.unwrap(); + assert!(!r.finished); + let r = f.session.exec("echo after", 5000).await.unwrap(); + assert_eq!(text(&r), "after\n"); +} + +#[tokio::test] +async fn poll_collects_the_rest_of_a_running_command() { + let mut f = start().await; + let r = f.session.exec("sleep 1; echo done", 100).await.unwrap(); + assert!(!r.finished); + let r = f.session.poll(5000).await.unwrap(); + assert!(r.finished); + assert_eq!(text(&r), "done\n"); +} + +#[tokio::test] +async fn restarts_after_the_shell_exits() { + let mut f = start().await; + f.session.exec("cd /", 5000).await.unwrap(); + let r = f.session.exec("exit 3", 5000).await.unwrap(); + assert!(r.finished); + assert_eq!(r.exit, None); + assert!(r.notes[0].contains("shell exited"), "{:?}", r.notes); + + let r = f.session.exec("pwd", 5000).await.unwrap(); + assert_eq!(text(&r), "/\n"); + assert!(r.notes[0].contains("new shell"), "{:?}", r.notes); +} + +#[tokio::test] +async fn removes_secrets_from_the_environment() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ShellConfig::new(dir.path(), dir.path().join("s")); + cfg.remove_env = vec!["HOME".into()]; + cfg.env = vec![("GA_EXTRA".into(), "yes".into())]; + let mut s = Session::start(cfg).await.unwrap(); + let r = s + .exec("echo ${HOME:-unset} $GA_EXTRA $PAGER", 5000) + .await + .unwrap(); + assert_eq!(text(&r), "unset yes cat\n"); +} + +#[tokio::test] +async fn large_output_is_capped() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ShellConfig::new(dir.path(), dir.path().join("s")); + cfg.max_buffer_bytes = 64 * 1024; + let mut s = Session::start(cfg).await.unwrap(); + let r = s.exec("seq 1 200000", 20_000).await.unwrap(); + assert!(r.finished); + assert_eq!(r.exit, Some(0)); + assert!(r.omitted_bytes > 1_000_000); + let t = text(&r); + assert!(t.starts_with("1\n2\n")); + assert!(t.ends_with("200000\n")); +} diff --git a/crates/tools/Cargo.toml b/crates/tools/Cargo.toml new file mode 100644 index 00000000..55fc80b2 --- /dev/null +++ b/crates/tools/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "gitagent-tools" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +gitagent-context.workspace = true +gitagent-loader.workspace = true +gitagent-provider.workspace = true +gitagent-record.workspace = true +gitagent-shaper.workspace = true +gitagent-terminal.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile = "3.27.0" +tokio.workspace = true diff --git a/crates/tools/src/exec.rs b/crates/tools/src/exec.rs new file mode 100644 index 00000000..582d5b3b --- /dev/null +++ b/crates/tools/src/exec.rs @@ -0,0 +1,178 @@ +use anyhow::Result; +use gitagent_provider::ToolCall; +use gitagent_shaper::Shaped; +use gitagent_terminal::{ExecResult, MAIN_SESSION}; + +use gitagent_context::MASK_PREFIX; + +use crate::{bool_arg, int_arg, str_arg, ToolContext, ToolOutput, MAX_YIELD_MS}; + +pub(crate) async fn run(call: &ToolCall, ctx: &mut ToolContext<'_>) -> Result { + let cmd = str_arg(call, "cmd")?; + let background = ctx.tools.write_stdin && bool_arg(call, "background")?; + let yield_ms = capped_yield( + int_arg(call, "yield_ms")?.unwrap_or(ctx.default_yield_ms), + ctx, + ); + if ctx.shaper.mentions_spill(cmd) { + ctx.counters.inc("spill_reads"); + } + let result = ctx.terminal.exec(cmd, yield_ms, background).await?; + if background { + ctx.counters.inc("background_sessions"); + } + Ok(finish(&result, ctx)) +} + +pub(crate) fn capped_yield(yield_ms: u64, ctx: &ToolContext<'_>) -> u64 { + let mut ms = yield_ms.min(MAX_YIELD_MS); + if let Some(left) = ctx.remaining { + ms = ms.min(left.as_millis() as u64); + } + ms +} + +/// Shapes a terminal result, records its counters and formats it for the model. +pub(crate) fn finish(r: &ExecResult, ctx: &mut ToolContext<'_>) -> ToolOutput { + if !r.finished { + ctx.counters.inc("exec_timeouts"); + } + if r.shell_restarted { + ctx.counters.inc("shell_restarts"); + } + let shaped = ctx.shaper.shape(&r.output); + ctx.counters.add("shaped_bytes_in", r.output.len() as u64); + ctx.counters + .add("shaped_bytes_out", shaped.text.len() as u64); + ctx.counters + .add("collapse_events", shaped.collapsed_runs as u64); + let mut out = ToolOutput::ok(format_result( + r, + &shaped, + ctx.tools.write_stdin, + ctx.minutes_left, + )); + out.finished = r.finished; + let status = match (r.finished, r.exit) { + (true, Some(code)) => format!("exit {code}"), + (true, None) => "shell exited".to_string(), + (false, _) => "still running".to_string(), + }; + out.stub = Some(match &shaped.spill_path { + Some(p) => format!( + "{MASK_PREFIX}; status was {status}, {} lines; full output: {}]", + shaped.total_lines, + p.display() + ), + None => format!("{MASK_PREFIX}; status was {status}]"), + }); + out +} + +pub(crate) fn format_result( + r: &ExecResult, + shaped: &Shaped, + sessions: bool, + minutes_left: Option, +) -> String { + let mut out = shaped.text.clone(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + let secs = r.elapsed.as_secs_f64(); + let label = if r.session > MAIN_SESSION { + format!("session {} · ", r.session) + } else { + String::new() + }; + // Point at the full output only when the model is not seeing all of it. + let mut full = match &shaped.spill_path { + Some(p) if shaped.hidden_lines > 0 || r.omitted_bytes > 0 => format!( + " · {} lines, full output: {}", + shaped.total_lines, + p.display() + ), + _ => String::new(), + }; + if let Some(m) = minutes_left { + full.push_str(&format!(" · {m} min left")); + } + let status = match (r.finished, r.exit) { + (true, Some(code)) => format!("[{label}exit {code} · {secs:.1}s{full}]"), + (true, None) => format!("[{label}shell exited · {secs:.1}s{full}]"), + (false, _) if sessions => { + let id = r.session.max(MAIN_SESSION); + format!( + "[session {id} still running after {secs:.1}s{full} · output so far is shown · use write_stdin with session_id {id} to wait, send input or interrupt]" + ) + } + (false, _) => format!( + "[still running after {secs:.1}s{full} · output so far is shown · the next exec call interrupts it]" + ), + }; + out.push_str(&status); + for note in &r.notes { + out.push('\n'); + out.push_str(note); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use gitagent_shaper::{shape, ShapeCfg}; + use std::time::Duration; + + fn result(output: &str, exit: Option, finished: bool, session: u32) -> ExecResult { + ExecResult { + session, + output: output.as_bytes().to_vec(), + exit, + finished, + omitted_bytes: 0, + elapsed: Duration::from_millis(1500), + shell_restarted: false, + notes: vec![], + } + } + + fn fmt(r: &ExecResult, sessions: bool) -> String { + let dir = tempfile::tempdir().unwrap(); + let cfg = ShapeCfg { + cap_tokens: 8000, + head_ratio: 0.3, + collapse_min_run: 3, + }; + let shaped = shape(&r.output, &cfg, dir.path(), 1); + format_result(r, &shaped, sessions, None) + .replace(&dir.path().display().to_string(), "SPILL") + } + + #[test] + fn formats_finished_and_running_commands() { + assert_eq!( + fmt(&result("hi\r\n", Some(0), true, 1), true), + "hi\n[exit 0 · 1.5s]" + ); + assert_eq!( + fmt(&result("", Some(2), true, 3), true), + "[session 3 · exit 2 · 1.5s]" + ); + assert!(fmt(&result("part", None, false, 2), true) + .starts_with("part\n[session 2 still running after 1.5s")); + assert!(fmt(&result("part", None, false, 1), false) + .ends_with("the next exec call interrupts it]")); + } + + #[test] + fn names_the_spill_file_when_lines_are_hidden() { + let many: String = (0..20).map(|i| format!("warning: item {i}\n")).collect(); + let out = fmt(&result(&many, Some(0), true, 1), true); + assert!(out.contains("[18 similar lines hidden]")); + assert!( + out.ends_with("[exit 0 · 1.5s · 20 lines, full output: SPILL/1.log]"), + "{out}" + ); + } +} diff --git a/crates/tools/src/image_read.rs b/crates/tools/src/image_read.rs new file mode 100644 index 00000000..e179ef93 --- /dev/null +++ b/crates/tools/src/image_read.rs @@ -0,0 +1,125 @@ +use std::path::Path; + +use anyhow::Result; +use gitagent_provider::{ImageData, ToolCall}; + +use crate::{str_arg, ToolOutput}; + +/// Providers reject larger images (about 5 MB once base64-encoded). +const MAX_IMAGE_BYTES: usize = 3_700_000; + +pub(crate) fn run(call: &ToolCall, cwd: &Path) -> Result { + let path = cwd.join(str_arg(call, "path")?); + let bytes = match std::fs::read(&path) { + Ok(b) => b, + Err(e) => { + return Ok(ToolOutput::error(format!( + "Cannot read {}: {e}", + path.display() + ))) + } + }; + let Some(media_type) = media_type(&bytes) else { + return Ok(ToolOutput::error(format!( + "{} is not a PNG, JPEG, GIF or WebP image. Convert it first (e.g. with ImageMagick or Python).", + path.display() + ))); + }; + if bytes.len() > MAX_IMAGE_BYTES { + return Ok(ToolOutput::error(format!( + "{} is {} bytes; images over {} bytes are rejected. Scale it down first.", + path.display(), + bytes.len(), + MAX_IMAGE_BYTES + ))); + } + let mut out = ToolOutput::ok(format!( + "Image {} ({media_type}, {} bytes).", + path.display(), + bytes.len() + )); + out.images.push(ImageData { + media_type: media_type.to_string(), + data: base64(&bytes), + }); + Ok(out) +} + +/// Detects the format from the file's first bytes. +fn media_type(b: &[u8]) -> Option<&'static str> { + if b.starts_with(b"\x89PNG\r\n\x1a\n") { + Some("image/png") + } else if b.starts_with(&[0xff, 0xd8, 0xff]) { + Some("image/jpeg") + } else if b.starts_with(b"GIF87a") || b.starts_with(b"GIF89a") { + Some("image/gif") + } else if b.len() >= 12 && &b[..4] == b"RIFF" && &b[8..12] == b"WEBP" { + Some("image/webp") + } else { + None + } +} + +/// Standard base64 with padding. +fn base64(bytes: &[u8]) -> String { + const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); + for chunk in bytes.chunks(3) { + let n = (u32::from(chunk[0]) << 16) + | (u32::from(*chunk.get(1).unwrap_or(&0)) << 8) + | u32::from(*chunk.get(2).unwrap_or(&0)); + out.push(TABLE[(n >> 18) as usize & 63] as char); + out.push(TABLE[(n >> 12) as usize & 63] as char); + out.push(if chunk.len() > 1 { + TABLE[(n >> 6) as usize & 63] as char + } else { + '=' + }); + out.push(if chunk.len() > 2 { + TABLE[n as usize & 63] as char + } else { + '=' + }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn call(path: &str) -> ToolCall { + ToolCall { + id: "c".into(), + name: "image_read".into(), + arguments: json!({"path": path}).as_object().unwrap().clone(), + } + } + + #[test] + fn base64_matches_the_standard_alphabet_and_padding() { + assert_eq!(base64(b""), ""); + assert_eq!(base64(b"f"), "Zg=="); + assert_eq!(base64(b"fo"), "Zm8="); + assert_eq!(base64(b"foo"), "Zm9v"); + assert_eq!(base64(b"foobar"), "Zm9vYmFy"); + assert_eq!(base64(&[0xff, 0xfe, 0xfd]), "//79"); + } + + #[test] + fn reads_images_and_rejects_other_files() { + let dir = tempfile::tempdir().unwrap(); + let png = b"\x89PNG\r\n\x1a\nrest-of-file"; + std::fs::write(dir.path().join("a.png"), png).unwrap(); + let out = run(&call("a.png"), dir.path()).unwrap(); + assert!(!out.is_error); + assert_eq!(out.images[0].media_type, "image/png"); + assert_eq!(out.images[0].data, base64(png)); + + std::fs::write(dir.path().join("b.txt"), "hello").unwrap(); + assert!(run(&call("b.txt"), dir.path()).unwrap().is_error); + assert!(run(&call("missing.png"), dir.path()).unwrap().is_error); + assert_eq!(media_type(b"RIFF\0\0\0\0WEBPVP8 "), Some("image/webp")); + } +} diff --git a/crates/tools/src/lib.rs b/crates/tools/src/lib.rs new file mode 100644 index 00000000..e2c4ffeb --- /dev/null +++ b/crates/tools/src/lib.rs @@ -0,0 +1,704 @@ +//! Tools offered to the model and their implementations. +//! +//! Tool definitions are part of the cached prompt prefix, so their text and +//! order must not change between tasks. + +mod exec; +mod image_read; +mod patch; +mod read_file; +mod skill; +mod stdin; +mod str_replace; +mod todo; +mod write_file; + +use std::time::Duration; + +use gitagent_loader::{AgentDef, EditTool, Skill, ToolsCfg}; +use gitagent_provider::{ImageData, ToolCall, ToolSpec}; +use gitagent_record::Counters; +use gitagent_shaper::Shaper; +use gitagent_terminal::Terminal; +use serde_json::{json, Value}; + +pub mod naming; + +pub const EXEC: &str = "exec"; +pub const WRITE_STDIN: &str = "write_stdin"; +pub const READ_FILE: &str = "read_file"; +pub const WRITE_FILE: &str = "write_file"; +pub const STR_REPLACE: &str = "str_replace"; +pub const APPLY_PATCH: &str = "apply_patch"; +pub const IMAGE_READ: &str = "image_read"; +pub const TODO: &str = "todo"; +pub const TASK_COMPLETE: &str = "task_complete"; +pub const SKILL: &str = "skill"; +pub const SUBAGENT: &str = "subagent"; + +/// Upper bound on how long one call may wait. +pub const MAX_YIELD_MS: u64 = 600_000; +/// Default wait after sending input with `write_stdin`. +pub const STDIN_YIELD_MS: u64 = 2_000; + +/// The tools for this run, in a fixed order. The set depends only on the +/// profile, never on the task. +pub fn specs(cfg: &ToolsCfg, edit_tool: EditTool) -> Vec { + let mut tools = vec![exec_spec(cfg.write_stdin)]; + if cfg.write_stdin { + tools.push(ToolSpec { + name: WRITE_STDIN.into(), + description: "Send input to a session whose command is still running, then return the \ + new output. With empty `chars` it just waits for more output. Write control keys \ + as escapes: \"\\x03\" is Ctrl-C (interrupt), \"\\x04\" is Ctrl-D (end of input), \ + \"\\e\" is Escape, \"\\n\" is Enter." + .into(), + input_schema: json!({ + "type": "object", + "properties": { + "session_id": {"type": "integer", "description": "The session to write to, as reported by exec (1 is the main shell)."}, + "chars": {"type": "string", "description": "Text and keys to send; empty to only wait for output."}, + "yield_ms": {"type": "integer", "description": "How long to wait for output, in milliseconds (default 2000 after input, 10000 when only waiting; max 600000)."} + }, + "required": ["session_id"] + }), + }); + } + tools.push(ToolSpec { + name: READ_FILE.into(), + description: "Read a text file. Returns numbered lines.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "Absolute path, or relative to the main shell's current directory."}, + "offset": {"type": "integer", "description": "First line to show, starting at 1 (default 1)."}, + "limit": {"type": "integer", "description": "Maximum number of lines to show (default 2000)."} + }, + "required": ["path"] + }), + }); + tools.push(ToolSpec { + name: WRITE_FILE.into(), + description: "Create a file with the given content, or replace a file completely. \ + Parent directories are created. To change part of an existing file, edit it with \ + a command instead." + .into(), + input_schema: json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "Absolute path, or relative to the main shell's current directory."}, + "content": {"type": "string", "description": "The full content of the file."} + }, + "required": ["path", "content"] + }), + }); + tools.push(edit_spec(edit_tool)); + if cfg.image_read { + tools.push(ToolSpec { + name: IMAGE_READ.into(), + description: "Look at an image file (PNG, JPEG, GIF or WebP, up to about 3.7 MB).".into(), + input_schema: json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "Absolute path, or relative to the main shell's current directory."} + }, + "required": ["path"] + }), + }); + } + if cfg.todo { + tools.push(ToolSpec { + name: TODO.into(), + description: "Keep a checklist for this task. Each call replaces the whole list; \ + use it to track requirements and what is left to verify." + .into(), + input_schema: json!({ + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "status": {"type": "string", "enum": ["pending", "in_progress", "done"]} + }, + "required": ["text"] + } + } + }, + "required": ["items"] + }), + }); + } + tools.push(ToolSpec { + name: TASK_COMPLETE.into(), + description: "Finish the task. Call this only after you have verified that every \ + requirement is met." + .into(), + input_schema: json!({ + "type": "object", + "properties": { + "summary": {"type": "string", "description": "One or two sentences on what was done and how it was verified."} + } + }), + }); + tools +} + +fn edit_spec(edit_tool: EditTool) -> ToolSpec { + match edit_tool { + EditTool::StrReplace => ToolSpec { + name: STR_REPLACE.into(), + description: "Edit a file by replacing an exact piece of text. `old_str` must match \ + the file exactly, including indentation, and appear once (or set replace_all). \ + Include a few surrounding lines to make it unique." + .into(), + input_schema: json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "Absolute path, or relative to the main shell's current directory."}, + "old_str": {"type": "string", "description": "The exact text to replace."}, + "new_str": {"type": "string", "description": "The text to put in its place."}, + "replace_all": {"type": "boolean", "description": "Replace every occurrence instead of requiring exactly one."} + }, + "required": ["path", "old_str", "new_str"] + }), + }, + EditTool::ApplyPatch => ToolSpec { + name: APPLY_PATCH.into(), + description: "Add, update, move or delete files with one patch:\n\ + *** Begin Patch\n\ + *** Add File: path/new.txt\n\ + +each line of the new file starts with +\n\ + *** Update File: path/app.py\n\ + @@ optional line to search for first, e.g. a function header\n\ + unchanged context line (starts with a space)\n\ + -removed line\n\ + +added line\n\ + *** Delete File: path/old.txt\n\ + *** End Patch\n\ + Add `*** Move to: new/path` right after an Update File line to rename. Give about \ + three lines of context around each change; use several @@ sections for changes far \ + apart. Paths are relative to the main shell's current directory." + .into(), + input_schema: json!({ + "type": "object", + "properties": { + "patch": {"type": "string", "description": "The whole patch, from *** Begin Patch to *** End Patch."} + }, + "required": ["patch"] + }), + }, + } +} + +fn exec_spec(sessions: bool) -> ToolSpec { + let mut props = json!({ + "cmd": { + "type": "string", + "description": "The command to run. Multi-line scripts and heredocs are fine." + }, + "yield_ms": { + "type": "integer", + "description": "How long to wait for the command to finish, in milliseconds (default 10000, max 600000). Use a longer wait for builds, installs and test suites." + } + }); + let description = if sessions { + props["background"] = json!({ + "type": "boolean", + "description": "Run in a new background session instead of the main shell (for servers, REPLs and watchers). It starts in the main shell's directory with its exported variables." + }); + "Run a shell command. By default it runs in the main shell (session 1), which keeps \ + its working directory, environment variables and shell state between calls. Returns \ + the output (stdout and stderr together) and the exit code. If the command is still \ + running after `yield_ms`, returns the output so far and the session id: use \ + write_stdin to wait for more output, send input or interrupt it. Running another \ + command in the main shell first interrupts a command still running there." + } else { + "Run a shell command in a persistent bash session. The working directory, \ + environment variables and shell state carry over between calls. Returns the output \ + (stdout and stderr together) and the exit code. If the command is still running \ + after `yield_ms`, returns the output so far; the next `exec` call first interrupts \ + it with Ctrl-C." + }; + ToolSpec { + name: EXEC.into(), + description: description.into(), + input_schema: json!({ + "type": "object", + "properties": props, + "required": ["cmd"] + }), + } +} + +/// What a tool call produced, ready to be sent back to the model. +#[derive(Debug, Clone, PartialEq)] +pub struct ToolOutput { + pub content: String, + pub is_error: bool, + /// Set by `task_complete`: the agent asked to finish. + pub complete: bool, + pub images: Vec, + /// False when a command was still running when its output was returned. + pub finished: bool, + /// What this result becomes if it is later hidden to save context. + pub stub: Option, +} + +impl ToolOutput { + fn ok(content: impl Into) -> Self { + ToolOutput { + content: content.into(), + is_error: false, + complete: false, + images: Vec::new(), + finished: true, + stub: None, + } + } + + fn error(content: impl Into) -> Self { + ToolOutput { + content: content.into(), + is_error: true, + complete: false, + images: Vec::new(), + finished: true, + stub: None, + } + } +} + +pub struct ToolContext<'a> { + pub terminal: &'a mut Terminal, + pub shaper: &'a mut Shaper, + pub counters: &'a mut Counters, + pub tools: &'a ToolsCfg, + pub edit_tool: EditTool, + pub skills: &'a [Skill], + pub default_yield_ms: u64, + /// Time left in the run, used to cap waits. + pub remaining: Option, + /// Shown in each command result when the time budget feature is on. + pub minutes_left: Option, + /// The model sees `.claude/` tool names (`tool_naming: sdk`): error messages + /// name tools and arguments the way the model knows them. + pub sdk_names: bool, +} + +/// Where tests are placed for grading; nothing may write there. +pub const PROTECTED_DIR: &str = "/tests"; + +pub async fn dispatch(call: &ToolCall, ctx: &mut ToolContext<'_>) -> ToolOutput { + if writes_protected_dir(call, ctx.terminal.cwd()) { + ctx.counters.inc("protected_writes_blocked"); + return ToolOutput::error(format!( + "Refused: nothing may be written under {PROTECTED_DIR}." + )); + } + let result = match call.name.as_str() { + EXEC => exec::run(call, ctx).await, + WRITE_STDIN if ctx.tools.write_stdin => stdin::run(call, ctx).await, + READ_FILE => { + if let Ok(path) = str_arg(call, "path") { + if ctx.shaper.mentions_spill(path) { + ctx.counters.inc("spill_reads"); + } + } + read_file::run(call, ctx.terminal.cwd()) + } + WRITE_FILE => write_file::run(call, ctx.terminal.cwd()), + STR_REPLACE if ctx.edit_tool == EditTool::StrReplace => run_str_replace(call, ctx), + APPLY_PATCH if ctx.edit_tool == EditTool::ApplyPatch => run_apply_patch(call, ctx), + SKILL if !ctx.skills.is_empty() => skill::run(call, ctx.skills), + IMAGE_READ if ctx.tools.image_read => image_read::run(call, ctx.terminal.cwd()), + TODO if ctx.tools.todo => todo::run(call), + TASK_COMPLETE => Ok(ToolOutput { + complete: true, + ..ToolOutput::ok("Task marked complete.") + }), + other => { + ctx.counters.inc("malformed_calls"); + let mut offered = specs(ctx.tools, ctx.edit_tool); + if ctx.sdk_names { + offered = naming::sdk_specs(offered); + } + let names: Vec = offered.into_iter().map(|t| t.name).collect(); + Ok(ToolOutput::error(format!( + "Unknown tool `{other}`. Available tools: {}.", + names.join(", ") + ))) + } + }; + result.unwrap_or_else(|e| { + if e.is::() { + ctx.counters.inc("malformed_calls"); + let shown = if ctx.sdk_names { + gitagent_loader::sdk::sdk_name(&call.name).to_string() + } else { + call.name.clone() + }; + return ToolOutput::error(format!( + "Invalid arguments for {shown}: {e:#}. {}", + expected_arguments(&call.name, ctx.tools, ctx.edit_tool, ctx.sdk_names) + )); + } + ToolOutput::error(format!("{e:#}")) + }) +} + +/// One line describing a tool's arguments, e.g. +/// "Expected arguments: cmd (string, required), yield_ms (integer)." +fn expected_arguments(tool: &str, cfg: &ToolsCfg, edit_tool: EditTool, sdk_names: bool) -> String { + let mut offered = specs(cfg, edit_tool); + let tool = if sdk_names { + offered = naming::sdk_specs(offered); + gitagent_loader::sdk::sdk_name(tool).to_string() + } else { + tool.to_string() + }; + let Some(spec) = offered.into_iter().find(|t| t.name == tool) else { + return String::new(); + }; + let required: Vec<&str> = spec.input_schema["required"] + .as_array() + .map(|r| r.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let props = spec.input_schema["properties"].as_object(); + let parts: Vec = props + .into_iter() + .flatten() + .map(|(name, schema)| { + let ty = schema["type"].as_str().unwrap_or("value"); + if required.contains(&name.as_str()) { + format!("{name} ({ty}, required)") + } else { + format!("{name} ({ty})") + } + }) + .collect(); + format!("Expected arguments: {}.", parts.join(", ")) +} + +fn run_str_replace(call: &ToolCall, ctx: &mut ToolContext<'_>) -> anyhow::Result { + let path = ctx.terminal.cwd().join(str_arg(call, "path")?); + let old = str_arg(call, "old_str")?; + let new = str_arg(call, "new_str")?; + let all = bool_arg(call, "replace_all")?; + Ok(match str_replace::edit(&path, old, new, all) { + Ok(e) => { + if e.fallback { + ctx.counters.inc("str_replace_fallbacks"); + } + ToolOutput::ok(e.message) + } + Err(msg) => ToolOutput::error(msg), + }) +} + +fn run_apply_patch(call: &ToolCall, ctx: &mut ToolContext<'_>) -> anyhow::Result { + Ok( + match patch::apply(str_arg(call, "patch")?, ctx.terminal.cwd()) { + Ok(done) => ToolOutput::ok(format!("Patch applied: {}.", done.join(", "))), + Err(msg) => { + ctx.counters.inc("patch_failures"); + ToolOutput::error(format!("Patch not applied (no files changed): {msg}")) + } + }, + ) +} + +/// The `skill` tool, offered when the config has skills. Added after the +/// fixed tools so the profile's tools keep their order. +pub fn skill_spec(skills: &[Skill]) -> ToolSpec { + let names: Vec<&str> = skills.iter().map(|s| s.name.as_str()).collect(); + ToolSpec { + name: SKILL.into(), + description: "Load the full instructions of a skill listed in the system prompt.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "name": {"type": "string", "enum": names, "description": "The skill's name."} + }, + "required": ["name"] + }), + } +} + +/// True when the call would write beneath `/tests`: file tools aimed there, +/// patches touching it, or shell commands that name it and write something. +fn writes_protected_dir(call: &ToolCall, cwd: &std::path::Path) -> bool { + let under = |p: &str| { + let full = cwd.join(p); + full.starts_with(PROTECTED_DIR) + }; + let arg = |k: &str| call.arguments.get(k).and_then(Value::as_str); + match call.name.as_str() { + WRITE_FILE | STR_REPLACE => arg("path").is_some_and(under), + APPLY_PATCH => arg("patch").is_some_and(|p| { + p.lines().any(|l| { + [ + "*** Add File: ", + "*** Update File: ", + "*** Delete File: ", + "*** Move to: ", + ] + .iter() + .any(|m| l.strip_prefix(m).is_some_and(|f| under(f.trim()))) + }) + }), + EXEC | WRITE_STDIN => { + let text = arg("cmd").or_else(|| arg("chars")).unwrap_or(""); + let mentions = regex_lite_find(text); + mentions + && (WRITE_WORDS.iter().any(|w| contains_word(text, w)) + || redirect_targets(text).any(|t| { + // After a `cd`, a relative target could resolve anywhere. + regex_lite_find(t) + || under(t) + || (!t.starts_with('/') && contains_word(text, "cd")) + })) + } + _ => false, + } +} + +const WRITE_WORDS: &[&str] = &[ + "tee", "cp", "mv", "rm", "mkdir", "touch", "ln", "chmod", "chown", "rsync", "install", "sed", + "truncate", "dd", "unzip", "tar", +]; + +/// The files a shell command redirects output into (`> f`, `>> f`, `&> f`, +/// `>| f`). Descriptor copies such as `2>&1` are not files and are skipped, so +/// `ls /tests 2>/dev/null` reads /tests without counting as a write to it. +fn redirect_targets(text: &str) -> impl Iterator { + text.match_indices('>').filter_map(move |(i, _)| { + if text[..i].ends_with('>') { + return None; // second `>` of `>>`, handled with the first + } + let rest = text[i + 1..].trim_start_matches(['>', '|']); + if rest.starts_with('&') || rest.starts_with('(') { + return None; // `2>&1`, `>&2`, or process substitution `>(...)` + } + let rest = rest.trim_start(); + let end = rest + .find(|c: char| c.is_whitespace() || matches!(c, ';' | '&' | '|' | ')')) + .unwrap_or(rest.len()); + Some(rest[..end].trim_matches(['"', '\''])) + }) +} + +/// Does `text` name `/tests` as a path (not, say, `/app/tests`)? +fn regex_lite_find(text: &str) -> bool { + text.match_indices(PROTECTED_DIR).any(|(i, _)| { + let before = text[..i].chars().last(); + let after = text[i + PROTECTED_DIR.len()..].chars().next(); + !before.is_some_and(|c| c.is_alphanumeric() || matches!(c, '/' | '.' | '_' | '-' | '~')) + && !after.is_some_and(|c| c.is_alphanumeric() || matches!(c, '_' | '-' | '.')) + }) +} + +fn contains_word(text: &str, word: &str) -> bool { + if word == ">" { + return text.contains('>'); + } + text.split(|c: char| !(c.is_alphanumeric() || c == '_' || c == '-')) + .any(|w| w == word) +} + +/// The `subagent` tool, offered when the profile enables sub-agents. +pub fn subagent_spec(agents: &[AgentDef]) -> ToolSpec { + let names: Vec<&str> = agents.iter().map(|a| a.name.as_str()).collect(); + let list: Vec = agents + .iter() + .map(|a| format!("- {}: {}", a.name, a.description)) + .collect(); + ToolSpec { + name: SUBAGENT.into(), + description: format!( + "Start a sub-agent for a self-contained piece of work. It starts with a fresh context \ + and its own shell in your current directory, sees only `prompt` (so include every path \ + and detail it needs), and returns only its final summary. Processes it starts stop when \ + it finishes. Agents:\n{}", + list.join("\n") + ), + input_schema: json!({ + "type": "object", + "properties": { + "description": {"type": "string", "description": "A few words on what the sub-agent will do."}, + "prompt": {"type": "string", "description": "The complete request for the sub-agent."}, + "agent": {"type": "string", "enum": names, "description": "Which agent to use (default: general)."} + }, + "required": ["description", "prompt"] + }), + } +} + +/// Invalid tool arguments from the model. +#[derive(Debug)] +struct ArgError(String); + +impl std::fmt::Display for ArgError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ArgError {} + +fn str_arg<'a>(call: &'a ToolCall, name: &str) -> anyhow::Result<&'a str> { + match call.arguments.get(name) { + Some(Value::String(s)) => Ok(s), + Some(_) => Err(ArgError(format!("`{name}` must be a string")).into()), + None => Err(ArgError(format!("missing required argument `{name}`")).into()), + } +} + +fn bool_arg(call: &ToolCall, name: &str) -> anyhow::Result { + match call.arguments.get(name) { + None | Some(Value::Null) => Ok(false), + Some(Value::Bool(b)) => Ok(*b), + Some(Value::String(s)) if s == "true" || s == "false" => Ok(s == "true"), + Some(_) => Err(ArgError(format!("`{name}` must be true or false")).into()), + } +} + +fn int_arg(call: &ToolCall, name: &str) -> anyhow::Result> { + match call.arguments.get(name) { + None | Some(Value::Null) => Ok(None), + Some(v) => v + .as_u64() + .or_else(|| v.as_f64().filter(|f| *f >= 0.0).map(|f| f as u64)) + .or_else(|| v.as_str().and_then(|s| s.trim().parse().ok())) + .map(Some) + .ok_or_else(|| ArgError(format!("`{name}` must be a non-negative integer")).into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn writes_under_tests_are_detected() { + let call = |name: &str, args: serde_json::Value| ToolCall { + id: "c".into(), + name: name.into(), + arguments: args.as_object().unwrap().clone(), + }; + let cwd = std::path::Path::new("/app"); + let blocked = |c: ToolCall| writes_protected_dir(&c, cwd); + assert!(blocked(call( + WRITE_FILE, + json!({"path": "/tests/test_x.py", "content": ""}) + ))); + assert!( + blocked(call( + WRITE_FILE, + json!({"path": "../tests/a", "content": ""}) + )) == std::path::Path::new("/app/../tests/a").starts_with("/tests") + ); + assert!(!blocked(call( + WRITE_FILE, + json!({"path": "tests/test_x.py", "content": ""}) + ))); + assert!(blocked(call( + EXEC, + json!({"cmd": "mkdir -p /tests && echo x > /tests/a"}) + ))); + assert!(blocked(call(EXEC, json!({"cmd": "cp out.txt /tests/"})))); + assert!(!blocked(call(EXEC, json!({"cmd": "ls /tests"})))); + // Reading /tests with an error redirect elsewhere is not a write to it. + assert!(!blocked(call( + EXEC, + json!({"cmd": "pip3 list 2>/dev/null; ls /tests 2>&1"}) + ))); + assert!(!blocked(call( + EXEC, + json!({"cmd": "cat /app/feal.py; ls /tests 2>/dev/null"}) + ))); + assert!(!blocked(call( + EXEC, + json!({"cmd": "ls /tests > /tmp/list.txt"}) + ))); + assert!(blocked(call(EXEC, json!({"cmd": "echo x > /tests/a"})))); + assert!(blocked(call( + EXEC, + json!({"cmd": "echo x >>/tests/a 2>&1"}) + ))); + assert!(blocked(call(EXEC, json!({"cmd": "make &> /tests/log"})))); + assert!(blocked(call( + EXEC, + json!({"cmd": "cd / && echo x > tests/a; ls /tests"}) + ))); + assert!(!blocked(call( + EXEC, + json!({"cmd": "grep '>' seq.fasta; ls /tests 2>/dev/null"}) + ))); + assert!(blocked(call( + EXEC, + json!({"cmd": "echo x > \"/tests/a b\""}) + ))); + assert!(!blocked(call( + EXEC, + json!({"cmd": "pytest /app/tests > log.txt"}) + ))); + assert!(!blocked(call( + EXEC, + json!({"cmd": "echo hi > /tmp/testsuite"}) + ))); + assert!(blocked(call( + APPLY_PATCH, + json!({"patch": "*** Begin Patch\n*** Add File: /tests/x\n+a\n*** End Patch"}) + ))); + } + + #[test] + fn specs_are_stable_and_follow_the_profile() { + let on = ToolsCfg { + write_stdin: true, + image_read: true, + todo: true, + subagents: true, + subagent_depth: 1, + max_concurrent_subagents: 4, + }; + let off = ToolsCfg { + write_stdin: false, + image_read: false, + todo: false, + subagents: false, + subagent_depth: 1, + max_concurrent_subagents: 4, + }; + let sr = EditTool::StrReplace; + let a = serde_json::to_string(&specs(&on, sr)).unwrap(); + assert_eq!(a, serde_json::to_string(&specs(&on, sr)).unwrap()); + let names = |c, e| specs(c, e).into_iter().map(|s| s.name).collect::>(); + assert_eq!( + names(&on, sr), + [ + EXEC, + WRITE_STDIN, + READ_FILE, + WRITE_FILE, + STR_REPLACE, + IMAGE_READ, + TODO, + TASK_COMPLETE + ] + ); + assert_eq!( + names(&off, EditTool::ApplyPatch), + [EXEC, READ_FILE, WRITE_FILE, APPLY_PATCH, TASK_COMPLETE] + ); + assert!(specs(&on, sr)[0].input_schema["properties"] + .get("background") + .is_some()); + assert!(specs(&off, sr)[0].input_schema["properties"] + .get("background") + .is_none()); + } +} diff --git a/crates/tools/src/naming.rs b/crates/tools/src/naming.rs new file mode 100644 index 00000000..e650b479 --- /dev/null +++ b/crates/tools/src/naming.rs @@ -0,0 +1,151 @@ +//! Tool definitions under the `.claude/` tool names (`tool_naming: sdk`): +//! `Bash` with `command`, `Read` with `file_path`, `Edit` with `old_string` +//! and `new_string`, … The model sees these; gitagent still runs its own +//! tools underneath (the loop maps each call back with +//! `gitagent_loader::sdk::gitagent_tool_for_call`). + +use gitagent_loader::sdk; +use gitagent_provider::ToolSpec; +use serde_json::{Map, Value}; + +/// The same tools, renamed for the model. `image_read` folds into `Read`, +/// which then also returns images; tools without a `.claude/` name keep theirs. +pub fn sdk_specs(specs: Vec) -> Vec { + let has_images = specs.iter().any(|t| t.name == crate::IMAGE_READ); + let mut out = Vec::new(); + for spec in specs { + if spec.name == crate::IMAGE_READ { + continue; + } + let renames = sdk::renames(&spec.name); + let mut description = spec.description.clone(); + for (ours, theirs) in renames { + description = replace_word(&description, ours, theirs); + } + if spec.name == crate::READ_FILE && has_images { + description.push_str( + " An image file (png, jpg, gif or webp) is returned as an image instead of text.", + ); + } + out.push(ToolSpec { + name: sdk::sdk_name(&spec.name).to_string(), + description, + input_schema: rename_schema(spec.input_schema, renames), + }); + } + out +} + +fn rename_schema(schema: Value, renames: &[(&str, &str)]) -> Value { + let Value::Object(mut obj) = schema else { + return schema; + }; + let to = |k: &str| { + renames + .iter() + .find(|(ours, _)| *ours == k) + .map(|(_, theirs)| theirs.to_string()) + .unwrap_or_else(|| k.to_string()) + }; + if let Some(Value::Object(props)) = obj.remove("properties") { + let mut renamed = Map::new(); + for (k, mut v) in props { + if let Some(d) = v.get("description").and_then(Value::as_str) { + let mut d = d.to_string(); + for (ours, theirs) in renames { + d = replace_word(&d, ours, theirs); + } + v["description"] = Value::String(d); + } + renamed.insert(to(&k), v); + } + obj.insert("properties".into(), Value::Object(renamed)); + } + if let Some(Value::Array(req)) = obj.remove("required") { + obj.insert( + "required".into(), + Value::Array( + req.into_iter() + .map(|r| match r { + Value::String(s) => Value::String(to(&s)), + other => other, + }) + .collect(), + ), + ); + } + Value::Object(obj) +} + +/// Replaces whole-word occurrences of an argument name in prose. +fn replace_word(text: &str, from: &str, to: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(i) = rest.find(from) { + let before = rest[..i].chars().next_back(); + let after = rest[i + from.len()..].chars().next(); + let is_word = |c: Option| c.is_some_and(|c| c.is_alphanumeric() || c == '_'); + out.push_str(&rest[..i]); + if is_word(before) || is_word(after) { + out.push_str(from); + } else { + out.push_str(to); + } + rest = &rest[i + from.len()..]; + } + out.push_str(rest); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use gitagent_loader::{EditTool, ToolsCfg}; + + #[test] + fn specs_take_sdk_names_and_argument_names() { + let cfg = ToolsCfg { + write_stdin: true, + image_read: true, + todo: false, + subagents: false, + subagent_depth: 1, + max_concurrent_subagents: 4, + }; + let specs = sdk_specs(crate::specs(&cfg, EditTool::StrReplace)); + let names: Vec<&str> = specs.iter().map(|t| t.name.as_str()).collect(); + assert_eq!( + names, + vec![ + "Bash", + "write_stdin", + "Read", + "Write", + "Edit", + "task_complete" + ] + ); + let bash = &specs[0]; + let props = bash.input_schema["properties"].as_object().unwrap(); + assert!(props.contains_key("command") && props.contains_key("timeout")); + assert!(!props.contains_key("cmd")); + assert_eq!( + bash.input_schema["required"], + serde_json::json!(["command"]) + ); + let read = &specs[2]; + assert!(read.input_schema["properties"].get("file_path").is_some()); + assert!(read.description.contains("returned as an image")); + let edit = &specs[4]; + let p = edit.input_schema["properties"].as_object().unwrap(); + assert!( + p.contains_key("old_string") + && p.contains_key("new_string") + && p.contains_key("file_path") + ); + assert_eq!( + replace_word("the cmd (not cmds) `cmd`", "cmd", "command"), + "the command (not cmds) `command`" + ); + } +} diff --git a/crates/tools/src/patch.rs b/crates/tools/src/patch.rs new file mode 100644 index 00000000..e5dafc52 --- /dev/null +++ b/crates/tools/src/patch.rs @@ -0,0 +1,562 @@ +//! The `apply_patch` tool: add, update, move or delete files with one patch. +//! +//! ```text +//! *** Begin Patch +//! *** Add File: path/new.txt +//! +first line of the new file +//! *** Update File: path/app.py +//! *** Move to: path/renamed.py +//! @@ def main(): +//! unchanged line +//! - removed line +//! + added line +//! *** Delete File: path/old.txt +//! *** End Patch +//! ``` + +use std::fs; +use std::path::{Path, PathBuf}; + +/// One file section of a patch. +enum Op { + Add { + path: String, + content: String, + }, + Delete(String), + Update { + path: String, + dest: Option, + chunks: Vec, + }, +} + +/// One `@@` section of an update. +#[derive(Default)] +struct Chunk { + ctx: Option, + old: Vec, + new: Vec, + eof: bool, +} + +impl Chunk { + fn has_lines(&self) -> bool { + !self.old.is_empty() || !self.new.is_empty() + } +} + +/// A file write planned while checking the patch; nothing touches the disk +/// until every section has been checked. +enum Write { + Put(PathBuf, String), + Remove(PathBuf), +} + +/// Apply `patch` under `cwd`. Returns one line per changed file, or a message +/// for the model explaining why nothing was changed. +pub(crate) fn apply(patch: &str, cwd: &Path) -> Result, String> { + let mut writes: Vec = Vec::new(); + let mut done = Vec::new(); + for op in parse(patch)? { + match op { + Op::Add { path, content } => { + writes.push(Write::Put(cwd.join(&path), content)); + done.push(format!("added {path}")); + } + Op::Delete(path) => { + let full = cwd.join(&path); + if !full.is_file() { + return Err(format!("cannot delete {path}: no such file")); + } + writes.push(Write::Remove(full)); + done.push(format!("deleted {path}")); + } + Op::Update { path, dest, chunks } => { + let full = cwd.join(&path); + // An earlier section of the same patch may already have + // rewritten this file. + let pending = writes.iter().rev().find_map(|w| match w { + Write::Put(p, text) if *p == full => Some(text.clone()), + _ => None, + }); + let text = match pending { + Some(text) => text, + None => { + fs::read_to_string(&full).map_err(|e| format!("cannot read {path}: {e}"))? + } + }; + let text = update(&text, &chunks).map_err(|e| format!("{path}: {e}"))?; + match dest.filter(|d| *d != path) { + Some(dest) => { + writes.push(Write::Put(cwd.join(&dest), text)); + writes.push(Write::Remove(full)); + done.push(format!("updated {path} and moved it to {dest}")); + } + None => { + writes.push(Write::Put(full, text)); + done.push(format!("updated {path}")); + } + } + } + } + } + for write in writes { + match write { + Write::Put(full, text) => { + if let Some(parent) = full.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("cannot write {}: {e}", full.display()))?; + } + fs::write(&full, text) + .map_err(|e| format!("cannot write {}: {e}", full.display()))?; + } + Write::Remove(full) => fs::remove_file(&full) + .map_err(|e| format!("cannot remove {}: {e}", full.display()))?, + } + } + Ok(done) +} + +/// The lines of a patch body with a read position. +struct Cursor<'a> { + lines: &'a [&'a str], + at: usize, +} + +impl<'a> Cursor<'a> { + fn peek(&self) -> Option<&'a str> { + self.lines.get(self.at).copied() + } + + fn next(&mut self) -> Option<&'a str> { + let line = self.peek()?; + self.at += 1; + Some(line) + } + + /// Patch line number of the body line at `index`, for error messages. + fn line_no(&self, index: usize) -> usize { + index + 2 + } +} + +/// The path after a `*** :` header, if `line` is one. +fn header(line: &str, kind: &str) -> Option { + let rest = line + .strip_prefix("*** ")? + .strip_prefix(kind)? + .strip_prefix(':')?; + Some(rest.trim().to_owned()) +} + +fn is_file_header(line: &str) -> bool { + ["Add File", "Update File", "Delete File"] + .iter() + .any(|kind| header(line, kind).is_some()) +} + +fn parse(patch: &str) -> Result, String> { + let lines: Vec<&str> = patch.trim().lines().collect(); + if lines.first().map(|l| l.trim_end()) != Some("*** Begin Patch") { + return Err("the patch must start with '*** Begin Patch'".into()); + } + if lines.len() < 2 || lines.last().map(|l| l.trim_end()) != Some("*** End Patch") { + return Err("the patch must end with '*** End Patch'".into()); + } + let mut cur = Cursor { + lines: &lines[1..lines.len() - 1], + at: 0, + }; + let mut ops = Vec::new(); + while let Some(line) = cur.next() { + let line = line.trim_end(); + if line.is_empty() { + continue; + } + if let Some(path) = header(line, "Add File") { + ops.push(parse_add(path, &mut cur)?); + } else if let Some(path) = header(line, "Delete File") { + ops.push(Op::Delete(path)); + } else if let Some(path) = header(line, "Update File") { + ops.push(parse_update(path, &mut cur)?); + } else { + return Err(format!( + "line {}: expected '*** Add File: ', '*** Update File: ' or '*** Delete File: ', got {line:?}", + cur.line_no(cur.at - 1) + )); + } + } + if ops.is_empty() { + return Err("the patch names no files".into()); + } + Ok(ops) +} + +fn parse_add(path: String, cur: &mut Cursor) -> Result { + let start = cur.at; + let mut body = Vec::new(); + while let Some(line) = cur.peek() { + if line.starts_with("*** ") { + break; + } + cur.next(); + body.push(line); + } + // Blank lines separating this section from the next are not content. + while body.last().is_some_and(|l| l.trim().is_empty()) { + body.pop(); + } + let mut content = String::new(); + for (i, line) in body.iter().enumerate() { + let Some(text) = line.strip_prefix('+') else { + return Err(format!( + "line {}: every line of an added file must start with '+', got {line:?}", + cur.line_no(start + i) + )); + }; + content.push_str(text); + content.push('\n'); + } + Ok(Op::Add { path, content }) +} + +fn parse_update(path: String, cur: &mut Cursor) -> Result { + let dest = cur.peek().and_then(|l| header(l, "Move to")); + if dest.is_some() { + cur.next(); + } + let mut chunks = Vec::new(); + let mut chunk = Chunk::default(); + while let Some(line) = cur.peek() { + if is_file_header(line) { + break; + } + cur.next(); + if let Some(ctx) = line.strip_prefix("@@") { + if chunk.has_lines() { + chunks.push(std::mem::take(&mut chunk)); + } + if !ctx.trim().is_empty() { + chunk.ctx = Some(ctx.trim().to_owned()); + } + } else if line.trim_end() == "*** End of File" { + chunk.eof = true; + if chunk.has_lines() { + chunks.push(std::mem::take(&mut chunk)); + } + } else if let Some(text) = line.strip_prefix(' ') { + chunk.old.push(text.to_owned()); + chunk.new.push(text.to_owned()); + } else if let Some(text) = line.strip_prefix('-') { + chunk.old.push(text.to_owned()); + } else if let Some(text) = line.strip_prefix('+') { + chunk.new.push(text.to_owned()); + } else if line.trim().is_empty() { + chunk.old.push(String::new()); + chunk.new.push(String::new()); + } else { + return Err(format!( + "line {}: {line:?} is not a patch line; start it with ' ', '-', '+' or '@@'", + cur.line_no(cur.at - 1) + )); + } + } + if chunk.has_lines() { + chunks.push(chunk); + } + if chunks.is_empty() && dest.is_none() { + return Err(format!("{path}: the update has no changes")); + } + Ok(Op::Update { path, dest, chunks }) +} + +/// Apply the chunks of one update to `text`. +fn update(text: &str, chunks: &[Chunk]) -> Result { + let mut lines: Vec = text.lines().map(str::to_owned).collect(); + let mut pos = 0; + for chunk in chunks { + let mut after_ctx = None; + if let Some(ctx) = &chunk.ctx { + let at = find(&lines, std::slice::from_ref(ctx), pos, false) + .ok_or_else(|| format!("could not find the context line \"{ctx}\""))?; + pos = at + 1; + after_ctx = Some(pos); + } + let (mut old, mut new) = (&chunk.old[..], &chunk.new[..]); + if old.is_empty() { + let at = match after_ctx { + Some(at) if !chunk.eof => at, + _ => lines.len(), + }; + lines.splice(at..at, new.iter().cloned()); + pos = at + new.len(); + continue; + } + let mut at = find(&lines, old, pos, chunk.eof); + // Models often end a chunk with a blank line that the file lacks. + if at.is_none() && old.last().is_some_and(|l| l.trim().is_empty()) { + old = &old[..old.len() - 1]; + if new.last().is_some_and(|l| l.trim().is_empty()) { + new = &new[..new.len() - 1]; + } + at = find(&lines, old, pos, chunk.eof); + } + let Some(at) = at else { + let shown: Vec<&str> = old.iter().take(10).map(String::as_str).collect(); + return Err(format!("could not find these lines:\n{}", shown.join("\n"))); + }; + lines.splice(at..at + old.len(), new.iter().cloned()); + pos = at + new.len(); + } + let mut out = lines.join("\n"); + if !out.is_empty() || text.ends_with('\n') { + out.push('\n'); + } + Ok(out) +} + +/// Index of `pat` in `lines` at or after `pos`, trying stricter comparisons +/// before lenient ones. With `eof`, a match ending at the last line wins. +fn find(lines: &[String], pat: &[String], pos: usize, eof: bool) -> Option { + if pat.is_empty() || pat.len() > lines.len() { + return None; + } + let last = lines.len() - pat.len(); + for mode in 0..4 { + let matches = |at: usize| { + lines[at..at + pat.len()] + .iter() + .zip(pat) + .all(|(l, p)| same(l, p, mode)) + }; + if eof && last >= pos && matches(last) { + return Some(last); + } + if let Some(at) = (pos..=last).find(|&at| matches(at)) { + return Some(at); + } + } + None +} + +fn same(a: &str, b: &str, mode: u8) -> bool { + match mode { + 0 => a == b, + 1 => a.trim_end() == b.trim_end(), + 2 => a.trim() == b.trim(), + _ => a.trim().chars().map(ascii).eq(b.trim().chars().map(ascii)), + } +} + +/// Typographic characters a model may emit in place of ASCII. +fn ascii(c: char) -> char { + match c { + '\u{2010}'..='\u{2015}' | '\u{2212}' => '-', + '\u{2018}'..='\u{201B}' => '\'', + '\u{201C}'..='\u{201F}' => '"', + '\u{00A0}' | '\u{2002}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}' => ' ', + c => c, + } +} + +#[cfg(test)] +mod extra_tests { + use super::*; + + fn dir_with(files: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + for (p, c) in files { + std::fs::write(dir.path().join(p), c).unwrap(); + } + dir + } + + fn read(dir: &tempfile::TempDir, p: &str) -> String { + std::fs::read_to_string(dir.path().join(p)).unwrap() + } + + #[test] + fn reports_a_missing_context_line() { + let dir = dir_with(&[("f.txt", "a\nb\n")]); + let patch = "*** Begin Patch\n*** Update File: f.txt\n@@ nope\n-b\n+c\n*** End Patch"; + let err = apply(patch, dir.path()).unwrap_err(); + assert!( + err.contains("could not find the context line \"nope\""), + "{err}" + ); + assert_eq!(read(&dir, "f.txt"), "a\nb\n"); + } + + #[test] + fn rejects_an_update_without_changes() { + let dir = dir_with(&[("f.txt", "a\n")]); + let patch = "*** Begin Patch\n*** Update File: f.txt\n*** End Patch"; + let err = apply(patch, dir.path()).unwrap_err(); + assert!(err.contains("has no changes"), "{err}"); + } + + #[test] + fn drops_a_trailing_blank_line_the_file_lacks() { + let dir = dir_with(&[("f.txt", "a\nb\n")]); + let patch = "*** Begin Patch\n*** Update File: f.txt\n a\n-b\n+c\n\n*** End Patch"; + apply(patch, dir.path()).unwrap(); + assert_eq!(read(&dir, "f.txt"), "a\nc\n"); + } + + #[test] + fn moves_without_chunks_and_updates_twice() { + let dir = dir_with(&[("f.txt", "a\n"), ("g.txt", "x\n")]); + let patch = "*** Begin Patch\n*** Update File: f.txt\n*** Move to: h.txt\n\ + *** Update File: g.txt\n-x\n+y\n*** Update File: g.txt\n-y\n+z\n*** End Patch"; + let done = apply(patch, dir.path()).unwrap(); + assert_eq!(done[0], "updated f.txt and moved it to h.txt"); + assert!(!dir.path().join("f.txt").exists()); + assert_eq!(read(&dir, "h.txt"), "a\n"); + assert_eq!(read(&dir, "g.txt"), "z\n"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dir_with(files: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + for (p, c) in files { + let full = dir.path().join(p); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(full, c).unwrap(); + } + dir + } + + fn read(dir: &tempfile::TempDir, p: &str) -> String { + std::fs::read_to_string(dir.path().join(p)).unwrap() + } + + #[test] + fn adds_updates_moves_and_deletes() { + let dir = dir_with(&[ + ("app.py", "def main():\n print(\"hi\")\n\nmain()\n"), + ("old.txt", "bye\n"), + ("move.txt", "a\nb\n"), + ]); + let patch = "*** Begin Patch +*** Add File: new/hello.txt ++hello ++world +*** Update File: app.py +@@ def main(): +- print(\"hi\") ++ print(\"hello\") +*** Delete File: old.txt +*** Update File: move.txt +*** Move to: moved/m.txt +@@ + a +-b ++c +*** End Patch"; + let summary = apply(patch, dir.path()).unwrap(); + assert_eq!(summary.len(), 4); + assert_eq!(read(&dir, "new/hello.txt"), "hello\nworld\n"); + assert_eq!( + read(&dir, "app.py"), + "def main():\n print(\"hello\")\n\nmain()\n" + ); + assert!(!dir.path().join("old.txt").exists()); + assert!(!dir.path().join("move.txt").exists()); + assert_eq!(read(&dir, "moved/m.txt"), "a\nc\n"); + } + + #[test] + fn matches_with_whitespace_and_typographic_differences() { + let dir = dir_with(&[("f.txt", "x = 1 \n y – z\ndone\n")]); + let patch = "*** Begin Patch +*** Update File: f.txt +-x = 1 +- y - z ++x = 2 ++y = z +*** End Patch"; + apply(patch, dir.path()).unwrap(); + assert_eq!(read(&dir, "f.txt"), "x = 2\ny = z\ndone\n"); + } + + #[test] + fn several_chunks_apply_in_order() { + let dir = dir_with(&[("f.txt", "a\nx\nb\nx\nc\n")]); + let patch = "*** Begin Patch +*** Update File: f.txt +@@ b +-x ++second +@@ + a +-x ++first +*** End Patch"; + // The second chunk searches from after the first; `a` is before it. + let err = apply(patch, dir.path()).unwrap_err(); + assert!(err.contains("could not find these lines"), "{err}"); + // Nothing was written when a chunk failed. + assert_eq!(read(&dir, "f.txt"), "a\nx\nb\nx\nc\n"); + + let patch = "*** Begin Patch +*** Update File: f.txt + a +-x ++first +@@ b +-x ++second +*** End Patch"; + apply(patch, dir.path()).unwrap(); + assert_eq!(read(&dir, "f.txt"), "a\nfirst\nb\nsecond\nc\n"); + } + + #[test] + fn end_of_file_and_pure_additions() { + let dir = dir_with(&[("f.txt", "x\ny\nx\n")]); + let patch = "*** Begin Patch +*** Update File: f.txt +-x ++last +*** End of File +*** End Patch"; + apply(patch, dir.path()).unwrap(); + assert_eq!(read(&dir, "f.txt"), "x\ny\nlast\n"); + let patch = "*** Begin Patch +*** Update File: f.txt ++appended +*** End Patch"; + apply(patch, dir.path()).unwrap(); + assert_eq!(read(&dir, "f.txt"), "x\ny\nlast\nappended\n"); + } + + #[test] + fn reports_malformed_patches() { + let dir = dir_with(&[]); + for (patch, msg) in [ + ("hello", "must start with"), + ("*** Begin Patch\n*** Add File: a\n+x", "must end with"), + ("*** Begin Patch\n*** End Patch", "no files"), + ( + "*** Begin Patch\n*** Add File: a\nx\n*** End Patch", + "must start with '+'", + ), + ( + "*** Begin Patch\n*** Frob File: a\n*** End Patch", + "expected '*** Add File", + ), + ( + "*** Begin Patch\n*** Delete File: none\n*** End Patch", + "no such file", + ), + ] { + let err = apply(patch, dir.path()).unwrap_err(); + assert!(err.contains(msg), "{patch:?}: {err}"); + } + } +} diff --git a/crates/tools/src/read_file.rs b/crates/tools/src/read_file.rs new file mode 100644 index 00000000..87bcfce5 --- /dev/null +++ b/crates/tools/src/read_file.rs @@ -0,0 +1,112 @@ +use std::fmt::Write as _; +use std::path::Path; + +use anyhow::Result; +use gitagent_provider::ToolCall; + +use crate::{int_arg, str_arg, ToolOutput}; + +const DEFAULT_LIMIT: u64 = 2000; +const MAX_LINE_CHARS: usize = 2000; +const MAX_TOTAL_CHARS: usize = 100_000; + +pub(crate) fn run(call: &ToolCall, cwd: &Path) -> Result { + let path = cwd.join(str_arg(call, "path")?); + let offset = int_arg(call, "offset")?.unwrap_or(1).max(1) as usize; + let limit = int_arg(call, "limit")?.unwrap_or(DEFAULT_LIMIT).max(1) as usize; + + let bytes = match std::fs::read(&path) { + Ok(b) => b, + Err(e) => { + return Ok(ToolOutput::error(format!( + "Cannot read {}: {e}", + path.display() + ))) + } + }; + if bytes[..bytes.len().min(8192)].contains(&0) { + return Ok(ToolOutput::error(format!( + "{} looks like a binary file ({} bytes). Inspect it with `exec` (e.g. `xxd | head`).", + path.display(), + bytes.len() + ))); + } + let text = String::from_utf8_lossy(&bytes); + let lines: Vec<&str> = text.lines().collect(); + if lines.is_empty() { + return Ok(ToolOutput::ok(format!("{} is empty.", path.display()))); + } + if offset > lines.len() { + return Ok(ToolOutput::error(format!( + "offset {offset} is past the end of {} ({} lines).", + path.display(), + lines.len() + ))); + } + + let mut out = String::new(); + let mut last = offset - 1; + for (i, line) in lines.iter().enumerate().skip(offset - 1).take(limit) { + if out.len() > MAX_TOTAL_CHARS { + break; + } + let line = line.strip_suffix('\r').unwrap_or(line); + if line.chars().count() > MAX_LINE_CHARS { + let cut: String = line.chars().take(MAX_LINE_CHARS).collect(); + let _ = writeln!(out, "{:>6}\t{cut} [line truncated]", i + 1); + } else { + let _ = writeln!(out, "{:>6}\t{line}", i + 1); + } + last = i + 1; + } + if last < lines.len() { + let _ = write!( + out, + "[showing lines {offset}-{last} of {}; use offset={} to continue]", + lines.len(), + last + 1 + ); + } + Ok(ToolOutput::ok(out.trim_end().to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn call(args: serde_json::Value) -> ToolCall { + ToolCall { + id: "c".into(), + name: "read_file".into(), + arguments: args.as_object().unwrap().clone(), + } + } + + #[test] + fn reads_numbered_lines_with_paging() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("a.txt"), "one\ntwo\r\nthree\n").unwrap(); + let out = run(&call(json!({"path": "a.txt"})), dir.path()).unwrap(); + assert_eq!(out.content, " 1\tone\n 2\ttwo\n 3\tthree"); + let out = run( + &call(json!({"path": "a.txt", "offset": 2, "limit": 1})), + dir.path(), + ) + .unwrap(); + assert_eq!( + out.content, + " 2\ttwo\n[showing lines 2-2 of 3; use offset=3 to continue]" + ); + } + + #[test] + fn reports_missing_and_binary_files() { + let dir = tempfile::tempdir().unwrap(); + let out = run(&call(json!({"path": "nope"})), dir.path()).unwrap(); + assert!(out.is_error); + std::fs::write(dir.path().join("b"), [0u8, 1, 2]).unwrap(); + let out = run(&call(json!({"path": "b"})), dir.path()).unwrap(); + assert!(out.is_error && out.content.contains("binary")); + } +} diff --git a/crates/tools/src/skill.rs b/crates/tools/src/skill.rs new file mode 100644 index 00000000..8e19faf7 --- /dev/null +++ b/crates/tools/src/skill.rs @@ -0,0 +1,30 @@ +use anyhow::Result; +use gitagent_loader::{split_frontmatter, Skill}; +use gitagent_provider::ToolCall; + +use crate::{str_arg, ToolOutput}; + +pub(crate) fn run(call: &ToolCall, skills: &[Skill]) -> Result { + let name = str_arg(call, "name")?; + let Some(skill) = skills.iter().find(|s| s.name == name) else { + let names: Vec<&str> = skills.iter().map(|s| s.name.as_str()).collect(); + return Ok(ToolOutput::error(format!( + "No skill named {name:?}. Skills: {}.", + names.join(", ") + ))); + }; + match std::fs::read_to_string(&skill.path) { + Ok(text) => { + let dir = skill + .path + .parent() + .map(|d| d.display().to_string()) + .unwrap_or_default(); + Ok(ToolOutput::ok(format!( + "Skill {name} (files next to it are in {dir}):\n\n{}", + split_frontmatter(&text).1.trim() + ))) + } + Err(e) => Ok(ToolOutput::error(format!("Cannot read skill {name}: {e}"))), + } +} diff --git a/crates/tools/src/stdin.rs b/crates/tools/src/stdin.rs new file mode 100644 index 00000000..dcfe7e2b --- /dev/null +++ b/crates/tools/src/stdin.rs @@ -0,0 +1,35 @@ +use anyhow::Result; +use gitagent_provider::ToolCall; +use gitagent_terminal::decode_keys; + +use crate::exec::{capped_yield, finish}; +use crate::{int_arg, ArgError, ToolContext, ToolOutput, STDIN_YIELD_MS}; + +pub(crate) async fn run(call: &ToolCall, ctx: &mut ToolContext<'_>) -> Result { + let id = int_arg(call, "session_id")? + .ok_or_else(|| ArgError("missing required argument `session_id`".into()))?; + let id = u32::try_from(id).map_err(|_| ArgError("`session_id` is too large".into()))?; + let chars = match call.arguments.get("chars") { + None | Some(serde_json::Value::Null) => "", + Some(serde_json::Value::String(s)) => s.as_str(), + Some(_) => return Err(ArgError("`chars` must be a string".into()).into()), + }; + let keys = decode_keys(chars); + let default_wait = if keys.is_empty() { + ctx.default_yield_ms + } else { + STDIN_YIELD_MS + }; + let yield_ms = capped_yield(int_arg(call, "yield_ms")?.unwrap_or(default_wait), ctx); + + if keys.is_empty() { + ctx.counters.inc("stdin_polls"); + } else { + ctx.counters.inc("stdin_writes"); + } + if keys.contains(&0x03) { + ctx.counters.inc("interrupts"); + } + let result = ctx.terminal.write_stdin(id, &keys, yield_ms).await?; + Ok(finish(&result, ctx)) +} diff --git a/crates/tools/src/str_replace.rs b/crates/tools/src/str_replace.rs new file mode 100644 index 00000000..385221c8 --- /dev/null +++ b/crates/tools/src/str_replace.rs @@ -0,0 +1,174 @@ +//! The `str_replace` tool: replace one exact occurrence of a text in a file. +//! +//! An exact match must be unique. When there is none, a whitespace-tolerant +//! match is tried: the same lines compared with surrounding whitespace +//! removed, and the replacement re-indented to the file's indentation. Those +//! fallback edits are counted separately. + +use std::path::Path; + +/// Result of an edit, for the tool output and counters. +pub(crate) struct Edited { + pub message: String, + pub fallback: bool, +} + +pub(crate) fn edit(path: &Path, old: &str, new: &str, replace_all: bool) -> Result { + if old.is_empty() { + return Err("`old_str` is empty; use write_file to create or replace a whole file".into()); + } + let text = std::fs::read_to_string(path) + .map_err(|e| format!("cannot read {}: {e}", path.display()))?; + + let matches: Vec = text.match_indices(old).map(|(i, _)| i).collect(); + let (updated, count, fallback) = match matches.len() { + 0 => { + let (start, end, indent) = fuzzy_find(&text, old).ok_or_else(|| { + format!( + "`old_str` was not found in {}. Read the file again and copy the exact text, including indentation.", + path.display() + ) + })?; + let replacement = reindent(new, &indent); + let mut out = String::with_capacity(text.len()); + out.push_str(&text[..start]); + out.push_str(&replacement); + out.push_str(&text[end..]); + (out, 1, true) + } + 1 => (text.replacen(old, new, 1), 1, false), + n if replace_all => (text.replace(old, new), n, false), + n => { + let lines: Vec = matches + .iter() + .map(|&i| (text[..i].matches('\n').count() + 1).to_string()) + .collect(); + return Err(format!( + "`old_str` appears {n} times in {} (lines {}). Include more surrounding text to make it unique, or set replace_all.", + path.display(), + lines.join(", ") + )); + } + }; + std::fs::write(path, &updated).map_err(|e| format!("cannot write {}: {e}", path.display()))?; + let at_line = updated + .find(new) + .map(|i| updated[..i].matches('\n').count() + 1); + let message = match (count, at_line) { + (1, Some(l)) => format!("Edited {} at line {l}.", path.display()), + (n, _) => format!("Edited {} ({n} replacements).", path.display()), + }; + Ok(Edited { + message: if fallback { + format!("{message} The text matched only after ignoring whitespace differences; check the result.") + } else { + message + }, + fallback, + }) +} + +/// Finds the lines of `old` in `text` comparing trimmed lines. Returns the +/// byte range of the matched lines and the indentation of the first one. +fn fuzzy_find(text: &str, old: &str) -> Option<(usize, usize, String)> { + let want: Vec<&str> = old.trim_matches('\n').lines().map(str::trim).collect(); + if want.iter().all(|l| l.is_empty()) { + return None; + } + let mut starts = Vec::new(); + let mut offset = 0; + for line in text.split_inclusive('\n') { + starts.push(offset); + offset += line.len(); + } + let lines: Vec<&str> = text.lines().collect(); + let mut found = None; + for i in 0..lines.len().saturating_sub(want.len() - 1) { + if (0..want.len()).all(|k| lines[i + k].trim() == want[k]) { + if found.is_some() { + return None; // ambiguous + } + found = Some(i); + } + } + let i = found?; + let last = i + want.len() - 1; + let start = starts[i]; + let end = starts[last] + lines[last].len(); + let indent: String = lines[i].chars().take_while(|c| c.is_whitespace()).collect(); + Some((start, end, indent)) +} + +/// Shifts `new` so its first line has `indent`, keeping relative indentation. +fn reindent(new: &str, indent: &str) -> String { + let lines: Vec<&str> = new.trim_matches('\n').lines().collect(); + let base = lines + .first() + .map(|l| l.len() - l.trim_start().len()) + .unwrap_or(0); + lines + .iter() + .map(|l| { + if l.trim().is_empty() { + String::new() + } else { + let own = l.len() - l.trim_start().len(); + format!( + "{indent}{}{}", + " ".repeat(own.saturating_sub(base)), + l.trim_start() + ) + } + }) + .collect::>() + .join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(content: &str) -> (tempfile::TempDir, std::path::PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("f.py"); + std::fs::write(&p, content).unwrap(); + (dir, p) + } + + #[test] + fn replaces_a_unique_exact_match() { + let (_d, p) = file("a = 1\nb = 2\n"); + let e = edit(&p, "b = 2", "b = 3", false).unwrap(); + assert!(!e.fallback); + assert!(e.message.ends_with("at line 2."), "{}", e.message); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "a = 1\nb = 3\n"); + } + + #[test] + fn errors_on_missing_or_ambiguous_text() { + let (_d, p) = file("x\ny\nx\n"); + let err = edit(&p, "x", "z", false).err().unwrap(); + assert!( + err.contains("appears 2 times") && err.contains("lines 1, 3"), + "{err}" + ); + let err = edit(&p, "nothing", "z", false).err().unwrap(); + assert!(err.contains("was not found"), "{err}"); + assert!(edit(&p, "", "z", false).is_err()); + let e = edit(&p, "x", "z", true).unwrap(); + assert!(e.message.contains("2 replacements")); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "z\ny\nz\n"); + } + + #[test] + fn falls_back_to_whitespace_tolerant_matching() { + let (_d, p) = file("def f():\n if x:\n return 1\n return 2\n"); + // Indentation in old_str is off, and new_str uses its own indentation. + let e = edit(&p, "if x:\n return 1", "if y:\n return 10", false).unwrap(); + assert!(e.fallback); + assert_eq!( + std::fs::read_to_string(&p).unwrap(), + "def f():\n if y:\n return 10\n return 2\n" + ); + } +} diff --git a/crates/tools/src/todo.rs b/crates/tools/src/todo.rs new file mode 100644 index 00000000..c4183454 --- /dev/null +++ b/crates/tools/src/todo.rs @@ -0,0 +1,69 @@ +//! A checklist the model keeps for itself. Each call replaces the whole list +//! and shows it back, so the current plan is always in the latest tool result. + +use std::fmt::Write as _; + +use anyhow::Result; +use gitagent_provider::ToolCall; +use serde_json::Value; + +use crate::{ArgError, ToolOutput}; + +pub(crate) fn run(call: &ToolCall) -> Result { + let Some(Value::Array(items)) = call.arguments.get("items") else { + return Err(ArgError("`items` must be a list of {text, status}".into()).into()); + }; + let mut out = String::new(); + let (mut done, mut total) = (0, 0); + for item in items { + let text = item["text"].as_str().unwrap_or("").trim(); + if text.is_empty() { + continue; + } + let status = item["status"].as_str().unwrap_or("pending"); + let mark = match status { + "done" => { + done += 1; + "[x]" + } + "in_progress" => "[~]", + _ => "[ ]", + }; + total += 1; + let _ = writeln!(out, "{mark} {text}"); + } + let _ = write!(out, "{done}/{total} done"); + Ok(ToolOutput::ok(out)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn renders_the_checklist() { + let call = ToolCall { + id: "c".into(), + name: "todo".into(), + arguments: json!({"items": [ + {"text": "build", "status": "done"}, + {"text": "test", "status": "in_progress"}, + {"text": "ship"}, + {"text": ""} + ]}) + .as_object() + .unwrap() + .clone(), + }; + assert_eq!( + run(&call).unwrap().content, + "[x] build\n[~] test\n[ ] ship\n1/3 done" + ); + let bad = ToolCall { + arguments: json!({"items": "x"}).as_object().unwrap().clone(), + ..call + }; + assert!(run(&bad).is_err()); + } +} diff --git a/crates/tools/src/write_file.rs b/crates/tools/src/write_file.rs new file mode 100644 index 00000000..f9160a61 --- /dev/null +++ b/crates/tools/src/write_file.rs @@ -0,0 +1,77 @@ +use std::path::Path; + +use anyhow::Result; +use gitagent_provider::ToolCall; + +use crate::{str_arg, ToolOutput}; + +pub(crate) fn run(call: &ToolCall, cwd: &Path) -> Result { + let path = cwd.join(str_arg(call, "path")?); + let content = str_arg(call, "content")?; + let existed = path.exists(); + if path.is_dir() { + return Ok(ToolOutput::error(format!( + "{} is a directory.", + path.display() + ))); + } + if let Some(parent) = path.parent() { + if let Err(e) = std::fs::create_dir_all(parent) { + return Ok(ToolOutput::error(format!( + "Cannot create {}: {e}", + parent.display() + ))); + } + } + if let Err(e) = std::fs::write(&path, content) { + return Ok(ToolOutput::error(format!( + "Cannot write {}: {e}", + path.display() + ))); + } + Ok(ToolOutput::ok(format!( + "{} {} ({} bytes, {} lines).", + if existed { "Overwrote" } else { "Created" }, + path.display(), + content.len(), + content.lines().count() + ))) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn call(args: serde_json::Value) -> ToolCall { + ToolCall { + id: "c".into(), + name: "write_file".into(), + arguments: args.as_object().unwrap().clone(), + } + } + + #[test] + fn creates_and_overwrites_files() { + let dir = tempfile::tempdir().unwrap(); + let out = run( + &call(json!({"path": "a/b/c.txt", "content": "x\ny\n"})), + dir.path(), + ) + .unwrap(); + assert!(out.content.starts_with("Created "), "{}", out.content); + assert!(out.content.ends_with("(4 bytes, 2 lines).")); + assert_eq!( + std::fs::read_to_string(dir.path().join("a/b/c.txt")).unwrap(), + "x\ny\n" + ); + let out = run( + &call(json!({"path": "a/b/c.txt", "content": ""})), + dir.path(), + ) + .unwrap(); + assert!(out.content.starts_with("Overwrote ")); + let out = run(&call(json!({"path": "a", "content": "z"})), dir.path()).unwrap(); + assert!(out.is_error); + } +} diff --git a/docs/DEVIATIONS.md b/docs/DEVIATIONS.md new file mode 100644 index 00000000..b2a8c1e9 --- /dev/null +++ b/docs/DEVIATIONS.md @@ -0,0 +1,185 @@ +# Deviations from BLUEPRINT + +Where an upstream API differs from the spec, we follow upstream and note it here. + +## Harbor CLI (harbor 0.24.0) + +The originally planned run command used flags that Harbor 0.24.0 does not have: + +| Planned | Harbor 0.24.0 | +|---|---| +| `--agent-import-path harbor.agent:GitAgent` | `-a harbor.agent:GitAgent` (`--agent` takes a built-in name or a `module.path:ClassName`) | +| `--task-ids ` | `-i ` repeated once per task (`--include-task-name`); `-x` excludes | +| `-k 3`, `--n-concurrent 8` | unchanged (`--n-attempts`, `--n-concurrent` / `-n`) | + +Task names in the dataset carry a `terminal-bench/` prefix (e.g. `terminal-bench/kv-store-grpc`); `bench/split.json` stores the full names. + +Working form (see the shim section below for the module path): + +``` +PYTHONPATH=$PWD harbor run -d terminal-bench/terminal-bench-2-1 -a gitagent_harbor.agent:GitAgent \ + -m --ak timeout_s= -k 3 -n 8 -o bench/jobs \ + $(jq -r '.dev[] | "-i " + .' bench/split.json) +``` + +## Harbor shim location (BLUEPRINT §5, §7) + +BLUEPRINT puts the shim at `harbor/agent.py`. A local top-level package named `harbor` would shadow the installed `harbor` package that the shim must import from (`harbor.agents.installed.base`). The shim lives at `gitagent_harbor/agent.py` instead and is loaded with `PYTHONPATH=$PWD harbor run -a gitagent_harbor.agent:GitAgent …`. Harbor does not put the working directory on `sys.path` by itself. + +## Trajectory and log paths (BLUEPRINT §7) + +BLUEPRINT says the shim copies `/tmp/gitagent/trajectory.atif.json`, `out/` and counters back. Harbor already syncs `/logs/agent/` in the container to the trial's `agent/` dir (bind mount or download after the run, including after a timeout), and backfills token usage from `agent/trajectory.json`. In `--bench`, the binary writes `trajectory.json`, `counters.json` and the spill dir under `/logs/agent/` directly, so no copy step can be lost on a timeout. Outside `--bench` the paths stay configurable. + +## ATIF version + +Harbor 0.24.0 validates ATIF `ATIF-v1.0`…`ATIF-v1.8`; we write `ATIF-v1.8`. Every model is `extra="forbid"`, so anything of ours (counters, guard events) goes in the `extra` fields, never as new keys. `step_id` must run 1, 2, 3, … and every observation result's `source_call_id` must match a tool call in the same step. + +## Time budget (TASKS M4) + +TASKS M4 says to read Harbor's task timeout from the environment or flags. Harbor 0.24.0 does not pass it to installed agents; it only enforces it from outside with `asyncio.wait_for`. The binary takes `--timeout-s`, and the shim fills it in from a shim kwarg (`--ak timeout_s=…`). Without it the binary uses `timeout_s` from `gitagent.yaml` (1800 s); if Harbor's limit is shorter, Harbor stops the run, and the trajectory written up to the last turn is kept. + +## API keys + +Harbor does not forward host environment variables into the container. The shim passes the provider key explicitly via `env=` on exec (from Harbor's resolved env, `--ae`, or the host environment). + +## Profile fields added (BLUEPRINT §6) + +Profiles also set `max_output_tokens` (providers require an output limit per call) and `context_window` (compaction in M4 needs to know how much room is left). Every profile field is required, so a profile always states its values. + +## Model string from Harbor + +Harbor passes models as `provider/model`; the shim converts this to the binary's `provider:model` form. + +## Scripted provider + +`--model scripted:` replays a fixed list of tool calls. It is used by the integration tests and for Harbor dry runs without a model API (`-m scripted/ --ak script=bench/fixtures/smoke.yaml`). It never decides anything from the task. + +## Anthropic: request surface (current Claude models) + +- **No temperature.** Current Claude models reject sampling parameters with a 400, so `profiles/anthropic.yaml` sets none (BLUEPRINT §6 shows `temperature: 1.0`). +- **Effort and thinking.** Thinking is adaptive and always on; `output_config.effort` is the control and is always sent explicitly (some models default to `medium`). Effort accepts `low`, `medium`, `high`, `xhigh`, `max`. +- **Output limit.** `max_output_tokens: 64000`, because thinking counts toward it. Responses are streamed so long turns do not hit HTTP timeouts. +- **Refusals.** A safety-classifier refusal ends the run with outcome `refused` (counter `refusals`). The profile enables the API's server-side refusal fallback (`fallbacks: "default"`), which re-runs a refused turn on a model Anthropic picks for that refusal category. Turns served by another model are counted in `fallback_turns`. Set `anthropic.refusal_fallback: false` for strict same-model comparisons. + +## Anthropic: thinking blocks bind the history (affects M4 masking and compaction) + +Thinking blocks are replayed verbatim and are bound to the exact conversation before them. Editing, removing or rewriting an earlier turn invalidates every later thinking block, and accounts created on or after 2026-08-31 get a 400 for it. The adapter sets `thinking.block_binding.prefix_mismatch_behavior` explicitly (profile `anthropic.thinking_binding`, default `error`) so any accidental edit fails loudly; the M1 loop is append-only. + +BLUEPRINT §5's batched masking replaces old tool results with stubs in place. That is such an edit. Options for M4, to be measured on the dev split: + +1. Anthropic's server-side tool-result clearing (context editing), which does not count as an edit. +2. Keep client-side masking at the batched checkpoints only, and send `drop_block` on those requests. Thinking blocks after the edit are dropped once per checkpoint. +3. Compaction that replaces the whole history with a summary and replays no earlier thinking blocks, as BLUEPRINT's structured compaction already does. + +Time-budget nudges (M4) must be appended (a mid-conversation `role: "system"` message or a text block after the tool results), never injected into an earlier turn. + +## HTTP client + +reqwest 0.12 with `rustls-tls` (ring plus bundled Mozilla root certificates) instead of 0.13, whose default TLS uses aws-lc and the system certificate store. Many task containers have no CA bundle; with bundled roots the static binary reaches the API from a container without `/etc/ssl` (checked in `busybox`). + +## Processes left running after the agent ends (Harbor shim, `--linger`) + +Harbor runs the task's tests after the agent process exits. Terminus 2 runs inside tmux, whose server keeps running in the container, so a server the agent started with `cmd &` is still up during the tests. A plain process would stop its shells on exit, and the terminal hang-up kills such jobs. Only `nohup`-style detached processes would survive. + +So in benchmark runs the shim starts gitagent detached with `--linger`. After the run, gitagent writes `done` (its exit code) next to the trajectory and keeps every shell open, draining output, until the shells exit or the container is removed. The shim returns as soon as `done` appears, so Harbor's agent timing ends with the agent's work. Outside the shim, gitagent still stops everything it started when it exits. + +## Output shaper (BLUEPRINT §5) + +- **Spill files** go to `{out-dir}/out/{n}.log`, next to the trajectory, so Harbor collects them. `spill_dir` was removed from `gitagent.yaml`. +- **Collapsing** keeps the first and the last line of a run of similar lines (the blueprint keeps only the first): the last line of a progress or build run is usually the one that matters. Any run of digits counts as one digit when comparing lines. `collapse_min_run: 0` turns collapsing off. +- **Token estimates** use about four bytes per token instead of a tokenizer library, to keep the binary small. The cap is approximate by design. +- **When no whole line fits** the budget (minified files, huge single lines), the shaper cuts characters from the middle instead. +- **Status line:** it names the spill file only when something was hidden, so short outputs stay short. The time-left part of the blueprint's footer comes with the time budget (M4). + +## OpenAI: Responses API, stateless + +- **Endpoint:** the adapter uses the Responses API (`POST /v1/responses`), not Chat Completions, because reasoning items can be carried across turns there. OpenAI-compatible endpoints for open-weight models (the `compat` family, M5) will use Chat Completions. +- **No stored state:** requests use `store: false` and replay the whole history every turn. Output items, including encrypted reasoning, go back unchanged, as OpenAI asks for reasoning models with tool calls. `include: ["reasoning.encrypted_content"]` is sent for older models; newer ones include it by default. +- **Prefix and effort:** the system prompt goes in `instructions` and stays byte-identical. Effort goes in `reasoning.effort`. Reasoning summaries (`openai.reasoning_summary: auto`) are recorded as `reasoning_content` in the trajectory. +- **Cache key:** every task sends the same `prompt_cache_key` (`openai.prompt_cache_key`), so requests that share the system prompt and tools are routed to the same cache. +- **No temperature** in `profiles/openai.yaml` (reasoning models do not take sampling parameters). +- **Errors:** quota and billing 429s are fatal, not retried; `context_length_exceeded` is reported as context overflow. +- **Not live-tested yet:** no OpenAI key so far. Request building, stream parsing and error handling are tested against a local mock server. + +## M3: snapshot and stop check are off by default + +Both exist and are tested, but `profiles/*.yaml` keep them off (`snapshot: false`, `stop.enabled: false`) until a dev-split comparison shows they help. `bench/profiles/anthropic-m3.yaml` switches them on for that comparison; the Harbor shim uploads a profile file given as `--ak profile=`. + +- **Snapshot** runs as one `sh -c` probe in its own process with a 15 s limit, not in the agent's shell, so it leaves no shell state and no command history. Memory comes from `/proc/meminfo`, because many task images lack `free`. On any failure the snapshot is left out and `snapshot_failed` is counted. +- **Stop check:** edits are `write_file` calls plus commands matching `stop.edit_patterns`. Redirections to `/dev/null` and between file descriptors are ignored. Verification is a *finished* command matching `stop.verify_patterns`. A command that both edits and verifies counts as verified. The rejection message names the unverified change and repeats the original task. The patterns are deliberately broad on the verify side (they include running programs and `git log`/`status`), so the check only catches completions with no evidence at all; tune them on the dev split. + +## M4: time budget, loop detector, masking, compaction + +All are profile switches. `bench/profiles/anthropic-m4.yaml` turns them on for measurement. In `profiles/*.yaml` everything is off except compaction. + +- **Reminders are appended, never injected.** Time reminders and loop warnings are added as a text block after the tool results of the turn, so earlier turns never change. Changed earlier turns would reset the prompt cache and invalidate Claude's thinking blocks. They also appear as user steps in the trajectory. +- **Time budget:** every command result shows `· N min left`. A reminder fires at `nudge_at` and a final call at `force_stop_at`. After the final call the stop check accepts `task_complete` without evidence, because there is no time left for a rejection to help. +- **Loop detector:** compares tool name, arguments and the first and last 20 output lines, with digit runs normalized. The effort increase after a warning (`loop_effort_bump`) is off by default: on Anthropic a top-level effort change resets the messages cache. +- **Masking** has three modes (`context.masking`): + - `client`: gitagent's own batched stubs, as in the blueprint. Each stub keeps the exit status, line count and spill file. + - `server`: Anthropic's context editing (`clear_tool_uses_20250919`) with trigger `mask_batch_tokens`, keep `keep_last` and clear-at-least `mask_batch_tokens / 2`. It is batched like the blueprint's masking and does not count as a history edit. + - `none`. + + The loader refuses `client` for a Claude profile whose `thinking_binding` is `error`, because client masking edits earlier turns. +- **Prompt size:** taken from the provider's reported input tokens after each response, plus an estimate (bytes / 4) for what was appended since. No tokenizer. +- **Compaction is on by default.** It only runs within `compact_at_tokens` of the context window, or after the provider reports a context overflow (then the turn is retried, at most twice per run). It replaces the history with the first message (task and snapshot) plus the model's structured summary: DONE, PENDING, KEY FILES, LAST FAILURE, RULED OUT, with the task repeated verbatim by us rather than by the model. If the summary request fails, a summary of the last 12 calls is built instead. No thinking blocks are carried across. +- **Cache lifetime:** `cache.ttl: 1h` asks Anthropic for the one-hour cache, which suits long command waits. The default stays `5m`. + +## M5: edit tools + +- **`apply_patch`** (OpenAI profile) is a compact patch-format edit tool in the style OpenAI models are trained on (Add, Update with optional Move, Delete, `@@` context, `*** End of File`). Lines are matched exactly first, then ignoring trailing whitespace, then both ends, then with typographic punctuation normalized. A patch is checked in full before any file is written, so a failing hunk changes nothing. +- **`str_replace`** (Anthropic, Google and compat profiles) requires one exact match, or `replace_all`. Errors list the line numbers of every match. With no exact match, a whitespace-tolerant fallback compares trimmed lines, applies only to a unique match, and re-indents the replacement to the file. It is counted as `str_replace_fallbacks`, and the tool result says so. +- Successful edits through either tool count as changes for the stop check. + +## M7: sandbox and /tests + +- **Where the sandbox applies:** to the agent's shell, not to gitagent itself. The shell is started through `gitagent __sandbox --write … -- `, which applies Landlock (writes allowed only under the working directory, the output directory, `/tmp`, `/var/tmp`, `/dev` and `~/.cache`; reads unrestricted) and a seccomp deny-list (mount, modules, reboot, keyrings, BPF, clock and host-name changes, …; denied calls fail with EPERM), then `exec`s the shell. Everything the shell starts inherits the restrictions, while gitagent can still write its records. +- **When:** `sandbox: auto` turns it on outside `--bench` on Linux. In `--bench` it is off: the task container is the isolation, and tasks legitimately write to `/etc`, `/usr` and other system paths. macOS has no sandbox; `sandbox: on` there is an error. +- **`/tests` writes are refused in every mode** by a built-in check in the tool dispatcher, not by Landlock. Landlock grants access only and cannot carve one directory out of an allowed tree. The check covers file tools and patches aimed at `/tests`, and shell commands that name `/tests` as a path together with a writing command. Refusals are counted as `protected_writes_blocked`. + +## Prompt: no clean-up directive + +`prompts/system.md` has no "leave no extra files or side effects" directive. Clean-up and minimal-state directives make agents delete state the result needs (services, generated files, installed packages), so the prompt says the opposite: do not undo what the result depends on. + +## Sub-agents (not in the blueprint; off by default) + +BLUEPRINT D2 is a single agent, and "multi-agent by default" is excluded. Sub-agents are therefore an opt-in profile switch (`tools.subagents`), to be measured on the dev split like any other feature. The design: + +- **Definitions:** a built-in `general` agent plus `agents/.md` files (`name`, `description`, `tools` allowlist, `max_turns`; the body holds the role instructions). +- **The `subagent` tool** runs a nested loop with a fresh history: the `prompt` string is the only handoff. The sub-agent gets its own shell in the parent's current directory, half of the remaining time, and no sub-agents of its own. +- **Results:** only its final summary returns to the parent. Its trajectory is written under `subagents//` and linked through ATIF `subagent_trajectory_ref`. Its tokens count toward the parent's totals. +- **Execution:** sub-agents run one at a time for now. + +## Continuous learning (not in the blueprint) + +The blueprint's agent has no memory across tasks. `gitagent learn` adds learning across runs, kept separate from benchmark runs: + +- **Process:** offline and reviewed. `gitagent learn --from …` reads each trial's `agent/trajectory.json` and the reward in its `result.json`, and nothing else. It never reads tests, verifier output or solutions. It asks the model (`prompts/learner.md`) for operations on a lesson store, `bench/lessons/lessons.json`: add, edit, upvote, downvote. A new lesson starts at importance 2, and a lesson is deleted when it reaches 0. +- **What is shown to the model:** an accepted and a rejected attempt at the same task go in one request so it can learn from the difference. Other trials go alone. +- **Leakage audit:** every added or edited lesson must pass. The audit rejects: + - the task name; + - paths, file names, quoted identifiers or numbers taken from the instruction; + - code blocks or long code spans; + - anything over 400 characters or under 5 words. +- **Evidence threshold:** importance rises only on the first vote from each task, so repeated runs of one task are not counted as wider evidence. A lesson reaches `prompts/lessons.md` only after attempts at `--min-tasks` (default 2) different tasks support it. The lessons file is capped at 3000 characters. +- **Integrity:** + - It learns from the dev split only. It refuses to run without `bench/split.json` unless given `--no-split`, which is meant only for tasks that will go in the dev split. + - The rendered file is reviewed and committed like any prompt change. Held-out runs use the committed, frozen file. The `lessons` profile switch turns it off. +- **Online mode (everyday use):** `learning: online` in `gitagent.yaml`, off by default and never in `--bench`. + - After each task the agent completes, the same learner reads that run's trajectory. It is told nobody checked the result. Each run counts as its own task for the threshold. + - Lessons go to `.gitagent/lessons.json` and `.gitagent/prompts/lessons.md` in the project. In repo mode they are a separate commit on the branch, so they are reviewed with the work. + - Lessons add up across layers: built-in, then config, project and home `prompts/lessons.md`, rather than one file overriding the others. +- **First pass (2026-10-15):** run on our 5 dev trials (fix-git ×4, kv-store-grpc ×1). It stored 3 lessons, each backed by one task, so none is shown yet. An earlier pass without the per-task rule produced 4 lessons about one task's domain (git merges) out of 5, which is why the threshold exists. + +## Split: stratified by category and difficulty, not Terminus 2 pass rate (TASKS M0) + +M0 asks for a 45/44 split stratified by category and Terminus 2 pass rate. No Terminus 2 baseline existed when development reached the point of running tasks (no amd64 bench host; the Mac runs tasks under emulation), and running the baseline on all 89 tasks there would have taken many hours. The owner chose (2026-10-15) to create the split from the task.toml `category` and `difficulty` instead, with a fixed seed (`scripts/make-split.py`, seed 20261015), forcing `fix-git` and `kv-store-grpc` into dev because their trajectories had been read. The split is committed and never changes. + +## `.claude/` project files and the library API (not in the blueprint) + +gitagent stays a from-scratch Rust binary that calls the model APIs itself (BLUEPRINT D1). On top of that it reads the `.claude/` project layout many teams already have, and offers a library API, delivered in steps: + +1. **File formats (done).** `.claude/agents/*.md`, `.claude/skills/` and the `hooks` object of `.claude/settings.json` (project and home) are read after gitagent's own layers. Agent files accept `tools` / `disallowedTools` (a comma string or YAML list), `maxTurns`, `model` and `effort`; the tool names used there are translated to gitagent's (`Bash` → `exec`, `Read` → `read_file` + `image_read`, `Edit` → `str_replace` + `apply_patch`, …) and names with no equivalent (`Grep`, `Glob`, `WebFetch`) are dropped. Settings hooks keep the protocol their scripts expect: `PreToolUse` / `PostToolUse` groups with `matcher` (`*`, `Edit|Write`, or a regex), `command` handlers with `timeout` and `${CLAUDE_PROJECT_DIR}`, stdin with `hook_event_name`, `tool_name`, `tool_input` (translated argument names), `tool_use_id` and `tool_response`, exit 2 blocks with stderr as the reason, other failures are non-blocking, and `hookSpecificOutput.permissionDecision` / `updatedInput` / `updatedToolOutput`, `decision: "block"` and `continue: false` are honoured. gitagent's own script hooks keep their stricter rule (any failure blocks) and run first. `--bench` ignores all of these layers, as it does every project file. +2. **Library API (done):** the `gitagent-sdk` crate. `query(prompt, AgentOptions)` runs the same loop and streams `Message::System` (`init`, `compact_boundary`), `Assistant`, `User` and `Result` (`session_id`, `parent_tool_use_id`, `num_turns`, `duration_ms`, `total_cost_usd`, `usage`, `modelUsage`, `permission_denials`); tool uses and results appear under the `.claude/` tool and argument names. `AgentOptions` covers `model`, `systemPrompt` / append, `cwd`, `allowedTools`, `disallowedTools`, `maxTurns`, `agents` (`AgentDefinition`), `hooks` (callbacks with `HookInput` / `HookOutput`) and `settingSources` (default none). Not offered: `permissionMode` (gitagent has no interactive permissions; the stream reports `bypassPermissions`), `resume`, `env`, `mcpServers`, `maxBudgetUsd`. Notes: `tool_result` content in the stream is cut at 2000 characters (the trajectory holds the full text); `duration_api_ms` counts time from each turn's start to the model's reply; `total_cost_usd` is 0 unless `prices` are given. The CLI gained `--output-format stream-json`, `--max-turns`, `--allowed-tools`, `--disallowed-tools` and `--append-system-prompt`. Provider construction moved from the CLI into `gitagent_provider::for_spec`. +3. **`.claude/` tool names offered to the model (built, off, unmeasured):** `tool_naming: sdk` in a profile renames the tool definitions the model sees (`Bash` / `command` / `timeout`, `Read` / `file_path` (images included; `image_read` folds into it), `Write`, `Edit` / `old_string` / `new_string`, `Agent`, `Skill`, `TodoWrite`); tools without such a name (`write_stdin`, `apply_patch`, `task_complete`) keep theirs. The loop maps each call back to gitagent's tool before hooks, guards and dispatch, counting `sdk_named_calls`; the trajectory records the name the model used. The default stays `gitagent` until a dev-split comparison says otherwise. +4. **Sub-agent features (built, off until measured):** a sub-agent's `model` (an alias such as `sonnet` or a model id, same provider as the parent) and `effort` are applied, each with a counter; sub-agents called in one reply run at the same time, `tools.max_concurrent_subagents` (default 4) at once, with their hooks asked first; `tools.subagent_depth` (default 1) says how deep they may nest; a sub-agent that hits `maxTurns` returns what it had as a partial result, not an error. Not done: background sub-agents and resuming one, and a dollar budget cap (`maxBudgetUsd`); gitagent's time budget bounds sub-agents instead (each gets half the time left). diff --git a/docs/INTEGRITY.md b/docs/INTEGRITY.md new file mode 100644 index 00000000..4e2375b4 --- /dev/null +++ b/docs/INTEGRITY.md @@ -0,0 +1,36 @@ +# Integrity + +How gitagent keeps benchmark results honest, and how to check it. + +## Rules and where they are enforced + +| Rule | Enforced by | +|---|---| +| No reading or writing of hidden tests; nothing written under `/tests` | The tool dispatcher refuses file tools, patches and shell commands that would write under `/tests`, in every mode (counter `protected_writes_blocked`). The system prompt also says so. Neither the agent loop nor the Harbor shim touches `/tests`. | +| No routing on task names, ids or hashes | Nothing in the code reads them. Behavior comes only from the instruction, the environment and the profile. `scripts/integrity-audit.sh` checks for task names. | +| Benchmark runs depend only on committed files | `--bench` ignores project and home config (`gitagent.yaml`, prompts, profiles, skills, hooks). Only the built-ins and explicit flags apply, and the Harbor shim uploads any profile it passes from the repository. | +| No network except the model API in `--bench` | No web tools exist. Telemetry export is disabled in `--bench`, and repo mode is refused with it. The agent's own shell commands run inside the task container, under Harbor's network settings. | +| No memory across tasks in `--bench` | Each run starts with a fresh history and shell. Interactive mode's follow-up tasks only continue within one session, and that mode is not available with `--bench`. Online learning (`learning: online`) lives in project config, which `--bench` ignores, and it never runs in `--bench`. | +| Learning only from dev tasks, frozen for held-out | `gitagent learn` reads only gitagent's own trajectories and each trial's reward, and only for tasks in the `dev` list of `bench/split.json`. Lessons that name the task or repeat its paths, identifiers, numbers or code are rejected. The result is a reviewed, committed `prompts/lessons.md`, which held-out runs use unchanged; nothing learns during a benchmark run. | +| No critic that rewrites answers | The stop check only accepts or rejects `task_complete`; it never changes the agent's work. | +| Held-out tasks unseen | `bench/split.json` (once created) stays fixed. Tasks whose trajectories were read during development (`fix-git`, `kv-store-grpc`) go in the dev split. | + +## Audit + +``` +scripts/integrity-audit.sh # task names from Harbor's local cache +scripts/integrity-audit.sh names.txt # or from a file, one name per line +``` + +It checks prompts, profiles, `gitagent.yaml`, the source, the Harbor shim and the benchmark profiles and fixtures for: + +1. any Terminal-Bench task name; +2. references to verifier code, reference solutions or test files (the `/tests` write guard and its tests, and the learner's reading of the reward from Harbor's `result.json`, are the only expected matches); +3. benchmark-specific wording in the system prompt as sent to the model (attribution comments are stripped before sending). + +### Results + +| Date | Commit | Task names checked | Result | +|---|---|---|---| +| 2026-10-14 | (this commit) | 89 (Terminal-Bench 2.1) | pass: no task names, no verifier or solution references, nothing benchmark-specific in the prompt | +| 2026-10-15 | (learner commit) | 89 (Terminal-Bench 2.1) | pass, after the learner was added | diff --git a/docs/LEARNING.md b/docs/LEARNING.md new file mode 100644 index 00000000..a6f7e303 --- /dev/null +++ b/docs/LEARNING.md @@ -0,0 +1,110 @@ +# How gitagent learns lessons + +gitagent improves between runs by turning its own finished attempts into short, +general lessons. A lesson reaches the agent only after it has been seen to hold on +at least two different tasks, a person has read the change, and the binary has +been rebuilt. Nothing is learned during a benchmark run. + +## The flow + +``` + Harbor job (dev tasks only) + └─ each trial: agent/trajectory.json + the reward in result.json + │ (never tests, verifier output or reference solutions) + ▼ + gitagent learn --from bench/jobs/ ... + 1. skip trials of tasks not in bench/split.json's "dev" list, and trials already learned from + 2. group the trials of one task into units: one passed and one failed attempt side by side when both exist, the rest one at a time + 3. show the learner model (prompts/learner.md) the unit and the current lesson list; + it answers with at most 3 operations: add, edit, upvote, downvote + 4. leakage audit: reject any added or edited text that carries task details + 5. apply the rest to the store + ▼ + bench/lessons/lessons.json ← the store: every lesson, with its evidence + ▼ render: lessons backed by ≥ 2 different tasks, by importance, up to 3,000 characters + prompts/lessons.md ← the lessons the agent will see (review this diff, then commit) + ▼ scripts/build-release.sh (the file is compiled into the binary) + target/*/release/gitagent ← appended to the system prompt as "# Lessons from earlier tasks" +``` + +## Where lessons live + +| File | What it holds | Who reads it | +|---|---|---| +| `bench/lessons/lessons.json` | Every lesson ever kept: text, importance, helpful/harmful counts, the **tasks** that added or confirmed it, and the **trials** it came from | `gitagent learn`, people reviewing | +| `prompts/lessons.md` | Only the lessons that passed the threshold, as a bullet list | Compiled into the binary; the agent sees it | +| `prompts/learner.md` | The learner model's instructions | `gitagent learn` | + +## The four operations + +The learner model sees one or two attempts at a task, whether the hidden checker +accepted them, and the current lessons. It may: + +| Operation | Effect on the store | +|---|---| +| `add` | New lesson, importance 2, `tasks` = this task. Skipped if it is ≥ 80% the same words as an existing lesson. | +| `edit` | Rewrites a lesson to be more accurate or general; counts as a vote from this task. | +| `upvote` | The attempt followed the lesson and it helped, or shows it is right: `helpful` + 1, vote from this task. | +| `downvote` | The attempt shows the lesson hurt or is wrong: importance − 1, `harmful` + 1. At importance 0 the lesson is deleted. | + +A vote from a task raises importance **only the first time that task votes**, and +adds the task to the lesson's `tasks` list. Running the same task many times +therefore cannot make a lesson look widely supported. + +## The leakage audit + +Every added or edited lesson is checked before it is stored. It is rejected if it +contains the task's name, a path or file name, a quoted identifier or a number +taken from the instruction, code, or is too long to be general advice (over 400 +characters; the learner is asked for at most 300). The learner is also told never to mention the task, +its files, ports, commands that only fit it, or its answer. Rejections are +counted in the learner's report (`ops_rejected`). + +## Which lessons reach the agent + +`prompts/lessons.md` is rebuilt from the store every time `gitagent learn` runs: + +1. keep lessons whose `tasks` list has **at least 2 different tasks** (`--min-tasks`, default 2); +2. sort them by **importance**, highest first; +3. add them one by one until the file would pass **3,000 characters** (`--max-chars`). + +A lesson seen on one task stays in the store, waiting for a second task to confirm +it. A lesson that is eligible but falls past the size limit also stays in the +store and can come back if its importance rises. The limit keeps the system +prompt short, and the same lessons go to every task, so the prompt stays +byte-identical and cached. + +## When lessons take effect + +The lessons are compiled into the binary, and Harbor uploads only the binary into +each task container. A change to `prompts/lessons.md` therefore reaches benchmark +runs only after: + +1. the diff of `prompts/lessons.md` has been read (no task details, sensible advice); +2. it is committed; +3. both binaries are rebuilt with `scripts/build-release.sh`. + +Outside benchmarks, `learning: online` in `gitagent.yaml` learns after each finished +task into the project's `.gitagent/prompts/lessons.md`, which later runs load on +top of the built-in lessons. It is never active with `--bench`. + +## Rules + +- Learn from dev tasks only. `gitagent learn` refuses trials of tasks outside + `bench/split.json`'s `dev` list, so held-out runs can never feed the lessons. +- The learner reads only gitagent's own trajectory and the reward; never tests, + verifier output or reference solutions. +- A lesson's support is the learner model's judgment that an attempt shows it, not + a measured cause. Whether the lessons help is measured separately, by running + rounds with and without them. + +## History so far + +| When | From | Store | In the prompt | Notes | +|---|---|---|---|---| +| Passes 1–4 (round r0, on the Mac) | the passes' dev trials | 53 | 9 | Learned after each pass; first lessons shown once a second task supported them | +| After round r1, run 1 (b15bb30) | 45 trials | 84 | 10 | 31 added, 72 upvoted, 10 edited, 2 rejected by the audit | +| After round r1, runs 2–3 (98b89c4) | 90 trials | 107 | 10 | 23 added, 183 upvoted, 16 edited, none rejected | +| Round r2 | — | — | 10 | Measured with the new lessons: 93.3% against r1's 94.1%, the same within noise | + +The held-out run uses r2's lessons, frozen; nothing is learned from it. diff --git a/gitagent.yaml b/gitagent.yaml new file mode 100644 index 00000000..88b29a1d --- /dev/null +++ b/gitagent.yaml @@ -0,0 +1,15 @@ +name: gitagent +# model: anthropic: # provider:model; --model overrides +fallback_models: [] +max_turns: 200 +timeout_s: 1800 # in --bench, --timeout-s from the Harbor shim wins +profile: auto # auto = pick by model family, or a profile name/path +sandbox: auto # auto | on | off; auto = on outside --bench on Linux +skills: true +hooks: true +learning: off # off | online: learn lessons into .gitagent/ after each finished task (never in --bench) +repo: + branch_prefix: gitagent/ + commit_every_turns: 0 + author: "gitagent " + push: false diff --git a/gitagent_harbor/__init__.py b/gitagent_harbor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gitagent_harbor/agent.py b/gitagent_harbor/agent.py new file mode 100644 index 00000000..c029a620 --- /dev/null +++ b/gitagent_harbor/agent.py @@ -0,0 +1,230 @@ +"""Harbor installed agent that runs the static gitagent binary in the task container. + +Usage (from the repository root, after building the musl binaries): + + PYTHONPATH=$PWD harbor run -d terminal-bench/terminal-bench-2-1 \ + -a gitagent_harbor.agent:GitAgent -m anthropic/ ... + +Agent kwargs (``--ak key=value``): + timeout_s wall-clock budget passed to ``--timeout-s``; set it a little + below Harbor's agent timeout so the run ends cleanly + profile built-in profile name, or a path to a profile YAML file on the + host (uploaded into the container), passed to ``--profile`` + binary_dir directory holding ``/release/gitagent`` (default: ./target) + script host path of a scripted-provider fixture; used with + ``-m scripted/`` for dry runs without a model API + +The agent never touches ``/tests``. +""" + +from __future__ import annotations + +import json +import shlex +from pathlib import Path + +from harbor.agents.capabilities import AgentCapabilities +from harbor.agents.installed.base import BaseInstalledAgent +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +REPO_ROOT = Path(__file__).resolve().parent.parent +REMOTE_DIR = "/opt/gitagent" +REMOTE_BIN = f"{REMOTE_DIR}/gitagent" +REMOTE_INSTRUCTION = f"{REMOTE_DIR}/instruction.md" +REMOTE_SCRIPT = f"{REMOTE_DIR}/script.yaml" +REMOTE_PROFILE = f"{REMOTE_DIR}/profile.yaml" + +TARGETS = { + "x86_64": "x86_64-unknown-linux-musl", + "amd64": "x86_64-unknown-linux-musl", + "aarch64": "aarch64-unknown-linux-musl", + "arm64": "aarch64-unknown-linux-musl", +} + +# Only the key for the model's provider is passed into the container. +PROVIDER_KEYS = { + "anthropic": ("ANTHROPIC_API_KEY",), + "openai": ("OPENAI_API_KEY",), + "google": ("GEMINI_API_KEY", "GOOGLE_API_KEY"), + "gemini": ("GEMINI_API_KEY", "GOOGLE_API_KEY"), + "openrouter": ("OPENROUTER_API_KEY",), +} + + +class GitAgent(BaseInstalledAgent): + capabilities = AgentCapabilities(atif=True) + + def __init__( + self, + *args, + timeout_s: int | float | str | None = None, + profile: str | None = None, + binary_dir: str | None = None, + script: str | None = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + self._timeout_s = int(float(timeout_s)) if timeout_s else None + self._profile = profile + self._binary_dir = Path(binary_dir) if binary_dir else REPO_ROOT / "target" + self._script = Path(script) if script else None + + @staticmethod + def name() -> str: + return "gitagent" + + async def install(self, environment: BaseEnvironment) -> None: + arch = (await self.exec_as_root(environment, "uname -m")).stdout.strip() + target = TARGETS.get(arch) + if target is None: + raise RuntimeError(f"unsupported container architecture: {arch!r}") + local = self._binary_dir / target / "release" / "gitagent" + if not local.is_file(): + raise FileNotFoundError( + f"{local} not found; run scripts/build-release.sh first" + ) + # Images come in both architectures, so every trial must get the same + # build: refuse a binary that is older than its sibling for the other + # architecture (someone rebuilt only one target). + for other in set(TARGETS.values()) - {target}: + sibling = self._binary_dir / other / "release" / "gitagent" + if sibling.is_file() and sibling.stat().st_mtime - local.stat().st_mtime > 600: + raise RuntimeError( + f"{local} is older than {sibling}; run scripts/build-release.sh " + "with no arguments so both architectures carry the same code" + ) + await self.exec_as_root(environment, f"mkdir -p {REMOTE_DIR}") + await environment.upload_file(local, REMOTE_BIN) + await self.exec_as_root(environment, f"chmod 755 {REMOTE_DIR} {REMOTE_BIN}") + version = await self.exec_as_agent(environment, f"{REMOTE_BIN} --version") + self._version = version.stdout.strip().split()[-1] + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + await self._upload_config_text( + environment, + content=self.render_instruction(instruction), + remote_path=REMOTE_INSTRUCTION, + filename="instruction.md", + ) + + provider, model = self._split_model() + env: dict[str, str] = {} + if provider == "scripted": + if self._script is None: + raise ValueError("model 'scripted/...' needs --ak script=") + await self._upload_config_text( + environment, + content=self._script.read_text(), + remote_path=REMOTE_SCRIPT, + filename="script.yaml", + ) + model_arg = f"scripted:{REMOTE_SCRIPT}" + else: + model_arg = f"{provider}:{model}" + upper = provider.upper().replace("-", "_") + # Known providers have fixed key names; other (OpenAI-compatible) + # endpoints use {NAME}_API_KEY and {NAME}_BASE_URL. + keys = PROVIDER_KEYS.get( + provider, (f"{upper}_API_KEY", f"{upper}_BASE_URL", "GITAGENT_API_KEY") + ) + for key in keys: + value = self._get_env(key) + if value: + env[key] = value + + logs = str(self.environment_logs_dir) + argv = [ + REMOTE_BIN, + "--bench", + "--instruction-file", + REMOTE_INSTRUCTION, + "--model", + model_arg, + "--out-dir", + logs, + ] + if self._profile: + profile = self._profile + local = Path(profile) + if local.suffix in (".yaml", ".yml") and local.is_file(): + await self._upload_config_text( + environment, + content=local.read_text(), + remote_path=REMOTE_PROFILE, + filename="profile.yaml", + ) + profile = REMOTE_PROFILE + argv += ["--profile", profile] + if self._timeout_s: + argv += ["--timeout-s", str(self._timeout_s)] + if getattr(self, "session_id", None): + argv += ["--session-id", str(self.session_id)] + argv.append("--linger") + + await self.exec_as_agent(environment, self._launch_command(argv, logs), env=env) + + @staticmethod + def _launch_command(argv: list[str], logs: str) -> str: + """Starts gitagent detached and returns once it writes its `done` file. + + gitagent keeps its shells open afterwards (--linger), so servers and + jobs the agent left running are still up when the tests run. + """ + q_logs = shlex.quote(logs) + done = shlex.quote(f"{logs}/done") + log_file = shlex.quote(f"{logs}/gitagent.log") + trajectory = shlex.quote(f"{logs}/trajectory.json") + # A launch is retried once when the process dies before it has + # written anything (no log, no trajectory): that is the emulator or + # the container failing at startup, not the agent's run. A crash + # after the run has started is reported as it is. + return ( + f"mkdir -p {q_logs} && rm -f {done}; " + f"if command -v nohup >/dev/null 2>&1; then n=nohup; else n=; fi; " + f"attempt=0; while :; do attempt=$((attempt+1)); " + f"$n {shlex.join(argv)} >{log_file} 2>&1 /dev/null; do sleep 1; done; " + f"if [ -f {done} ]; then break; fi; " + f"wait $pid; code=$?; " + f"if [ $attempt -lt 2 ] && [ $code -ge 128 ] && [ ! -s {log_file} ] && [ ! -f {trajectory} ]; then " + f'echo "gitagent died with status $code before starting; launching again" >&2; sleep 2; continue; fi; ' + f"cat {log_file}; exit $code; done; " + f"cat {log_file}; " + f'exit "$(head -n 1 {done})"' + ) + + def populate_context_post_run(self, context: AgentContext) -> None: + path = self.logs_dir / "trajectory.json" + if not path.is_file(): + return + trajectory = json.loads(path.read_text()) + # Summed per step so a run cut off by a timeout still reports usage. + prompt = completion = cached = 0 + for step in trajectory.get("steps", []): + metrics = step.get("metrics") or {} + prompt += metrics.get("prompt_tokens") or 0 + completion += metrics.get("completion_tokens") or 0 + cached += metrics.get("cached_tokens") or 0 + context.n_input_tokens = prompt + context.n_output_tokens = completion + context.n_cache_tokens = cached + + metadata = {"outcome": (trajectory.get("extra") or {}).get("outcome")} + counters = self.logs_dir / "counters.json" + if counters.is_file(): + metadata["counters"] = json.loads(counters.read_text()) + context.metadata = metadata + + def _split_model(self) -> tuple[str, str]: + if not self.model_name: + raise ValueError("gitagent needs a model: -m provider/model") + provider, _, model = self.model_name.partition("/") + if not model: + raise ValueError(f"model must look like provider/model, got {self.model_name!r}") + return provider, model diff --git a/profiles/anthropic.yaml b/profiles/anthropic.yaml new file mode 100644 index 00000000..895f1de5 --- /dev/null +++ b/profiles/anthropic.yaml @@ -0,0 +1,62 @@ +family: anthropic +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: str_replace +effort: high +# No temperature: current Claude models reject sampling parameters. +max_output_tokens: 64000 # thinking counts toward this limit +context_window: 1000000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: false # minutes-left footer, nudge, final call (M4; off until measured) + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: false # warn after loop_repeats identical calls and results (M4) + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: none # none | client | server (server: Anthropic clears old results) + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: false # environment snapshot in the first message (M3; off until measured) +stop: # stop check (M3; off until measured) + enabled: false + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [system, last_two_messages], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } +anthropic: + thinking_display: summarized + thinking_binding: error + refusal_fallback: true diff --git a/profiles/compat.yaml b/profiles/compat.yaml new file mode 100644 index 00000000..0831d674 --- /dev/null +++ b/profiles/compat.yaml @@ -0,0 +1,58 @@ +family: compat +tool_calling: native_with_text_fallback +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: str_replace +effort: medium +temperature: 0.7 +max_output_tokens: 16000 +context_window: 128000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 6000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: false # minutes-left footer, nudge, final call (M4; off until measured) + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: false # warn after loop_repeats identical calls and results (M4) + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: none # none | client | server (server: Anthropic clears old results) + mask_batch_tokens: 40000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: false # environment snapshot in the first message (M3; off until measured) +stop: # stop check (M3; off until measured) + enabled: false + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [], ttl: 5m } +tools: { write_stdin: true, image_read: false, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } diff --git a/profiles/google.yaml b/profiles/google.yaml new file mode 100644 index 00000000..9c71f0d9 --- /dev/null +++ b/profiles/google.yaml @@ -0,0 +1,58 @@ +family: google +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: str_replace +effort: high +temperature: 1.0 +max_output_tokens: 32000 +context_window: 1000000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: false # minutes-left footer, nudge, final call (M4; off until measured) + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: false # warn after loop_repeats identical calls and results (M4) + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: none # none | client | server (server: Anthropic clears old results) + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: false # environment snapshot in the first message (M3; off until measured) +stop: # stop check (M3; off until measured) + enabled: false + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } diff --git a/profiles/openai.yaml b/profiles/openai.yaml new file mode 100644 index 00000000..e33be150 --- /dev/null +++ b/profiles/openai.yaml @@ -0,0 +1,61 @@ +family: openai +tool_calling: native +tool_naming: gitagent # gitagent | sdk: the tool names the model sees (sdk = Bash, Read, Edit, …; measure before changing) +edit_tool: apply_patch +effort: high +# No temperature: reasoning models do not take sampling parameters. +max_output_tokens: 64000 # reasoning counts toward this limit +context_window: 400000 +exec_default_yield_ms: 10000 +shape: { cap_tokens: 8000, head_ratio: 0.3, collapse_min_run: 3 } +guards: + time_budget: false # minutes-left footer, nudge, final call (M4; off until measured) + nudge_at: 0.7 + force_stop_at: 0.9 + loop_detector: false # warn after loop_repeats identical calls and results (M4) + loop_repeats: 3 + loop_effort_bump: false # raise effort for one call after a loop warning +context: + masking: none # none | client | server (server: Anthropic clears old results) + mask_batch_tokens: 50000 + keep_last: 8 + compaction: true # summarize the history when it nears the context window + compact_at_tokens: 16000 +lessons: true # append prompts/lessons.md (learned from dev runs) +snapshot: false # environment snapshot in the first message (M3; off until measured) +stop: # stop check (M3; off until measured) + enabled: false + max_rejections: 2 + # A command matching one of these changed something (after removing + # redirections to /dev/null and between file descriptors). + edit_patterns: + - '\bsed\s+(-\w+\s+)*-i' + - '\bperl\s+-\w*i' + - '>{1,2}\s*[^\s&|>]' + - '\btee\b' + - '\b(mv|cp|rm|ln|chmod|patch)\s' + - '\bgit\s+(apply|am|checkout|switch|merge|cherry-pick|rebase|reset|restore|revert|commit|stash\s+pop)\b' + - '\b(pip3?|npm|pnpm|yarn|cargo|apt-get|apt|apk|yum|dnf|gem|conda)\s+(install|add)\b' + # A finished command matching one of these checked something. + verify_patterns: + - '\b(pytest|py\.test|tox|nox)\b' + - '\bpython3?\s+-m\s+(pytest|unittest|doctest)\b' + - '\b(npm|pnpm|yarn)\s+(run\s+)?test\b' + - '\b(cargo|go)\s+(test|run)\b' + - '\bmake\b' + - '\b(ctest|jest|mocha|vitest|rspec|phpunit)\b' + - '\b(mvn|gradle|gradlew)\b.*\btest\b' + - '\b(curl|wget|nc|ncat)\s' + - '\bpython3?\s+(-c|\S+\.py)' + - '\bnode\s+(-e|\S+\.m?js)' + - '(^|[;&|(]\s*|\s)\./[\w.-]+' + - '\b(bash|sh)\s+\S+\.sh\b' + - '\b(diff|cmp)\s' + - '\b(psql|mysql|sqlite3)\s' + - '\bgit\s+(clone|push|log|status|diff|show)\b' + - '\b(java|ruby|perl|php|Rscript|julia|lua)\s+\S+' +cache: { breakpoints: [], ttl: 5m } +tools: { write_stdin: true, image_read: true, todo: false, subagents: false, subagent_depth: 1, max_concurrent_subagents: 4 } +openai: + reasoning_summary: auto + prompt_cache_key: gitagent-v2 diff --git a/prompts/learner.md b/prompts/learner.md new file mode 100644 index 00000000..386fcd0a --- /dev/null +++ b/prompts/learner.md @@ -0,0 +1,22 @@ + +You keep a short list of lessons for a terminal agent that solves tasks in Linux containers. The lessons are added to the agent's system prompt when it works on future, different tasks. You are shown one or two finished attempts: the agent's own trajectory and whether a hidden checker accepted the result. You never see the checker itself. + +Reply with only a JSON object of the form {"ops": [...]}, holding at most 3 operations: +- {"op": "add", "text": "..."}: a new lesson. +- {"op": "edit", "id": N, "text": "..."}: rewrite lesson N to make it more accurate or more general. +- {"op": "upvote", "id": N}: the attempt followed lesson N and it helped, or the attempt shows lesson N is right. +- {"op": "downvote", "id": N}: the attempt followed lesson N and it hurt, or the attempt shows lesson N is wrong or useless. + +A lesson is one or two sentences of general advice about how to work, useful across many different tasks, and at most 300 characters; longer lessons are discarded unread. +- Never mention this task: no task names, file names, paths, ports, numbers, identifiers, commands that only fit this task, or its answer. +- Prefer habits that carry across domains (how to explore, check, debug, verify and keep the result in place) over facts about one program. +- Only claim what the trajectory shows. A rejected attempt tells you something was wrong, not what; if the cause is not visible in the trajectory, add nothing. +- When an attempt was accepted, learn from what made it efficient or careful; when one was accepted and one rejected, learn from the difference. +- Upvote or edit an existing lesson rather than adding a near-duplicate. +- Reply {"ops": []} when there is nothing general to learn. diff --git a/prompts/lessons.md b/prompts/lessons.md new file mode 100644 index 00000000..a3827648 --- /dev/null +++ b/prompts/lessons.md @@ -0,0 +1,17 @@ + +- Order your endgame: finish edits and stop jobs you started, then run the final verification as its own plain command after the last write, not chained with cleanup, edits or guard tests (test -s, command -v ...). Prefer the task's stated check run verbatim and unpiped; it must write nothing. +- Under a tight time budget, start slow work (installs, builds, a timed reference run of the original) in the background at once, logging output and taking stdin from /dev/null, and keep developing meanwhile. Install only what is needed, and poll the log and job state instead of waiting blindly. +- Make the final check exit 0 only when the result is correct: print OK from inside the check (or chain with &&), never as a separate trailing line. In expect scripts a timed-out expect continues silently, so add a timeout branch that exits nonzero. +- After writing a result file, reload it in a separate fresh process and recompute every required property straight from the stated definitions, such as size, type, validity and target values, rather than trusting the variables that produced it. +- Run the task's exact commands verbatim from a neutral directory and from the project folder, since paths resolve relative to the cwd. If your deliverable misses inputs from elsewhere, also look next to itself. A provided program that only works from its own folder is its convention, not your bug. +- Before declaring completion, stop background processes you started that the result does not need. Leave files in the task's directories in place; clear only your own scratch files in /tmp, and only if they could interfere. If ps/pgrep/pkill are missing, read /proc/*/cmdline to find, confirm or kill processes by command line. +- Before building or running anything that might write over provided input files, back them up and work in a scratch directory, then reproduce the stated acceptance check locally (the exact build command, the exact size or format limits, and the scoring metric against the reference) before declaring completion. +- When a hidden harness will call your function or run your deliverable, write your own driver that invokes it the same way across many random inputs plus edge cases (zero, extremes, boundaries), compares against a simple reference, and times it, before declaring completion. +- If a runtime or tool needed to run or check the deliverable is missing, install it (the distro package for a library also pulls in its interpreter) and run every required invocation for real, rather than shipping something untested. Note any version mismatch in your summary. +- Run the final verification as a command whose own text shows the check (a named script or a one-liner, not a heredoc piped into an interpreter). Give it a long enough wait to finish in that same call, since a result collected later by polling may not count as verifying your last change. diff --git a/prompts/system.md b/prompts/system.md new file mode 100644 index 00000000..6714274a --- /dev/null +++ b/prompts/system.md @@ -0,0 +1,37 @@ +# Role and environment + +You are an autonomous agent completing a task in a Linux environment. The task arrives in the first user message. + +- You act only through the tools you are given. Each tool's own description says how it behaves. +- Shell commands run in one persistent bash session: the working directory and environment variables carry over from one command to the next. +- There is no human in the loop. Nobody will answer a question, clarify a requirement or approve a step. Complete the whole task without asking. +- You cannot see images or hear audio directly. Inspect images, audio, video and other binary files with programs and the available tools. + +# Method + +Work in this order: + +1. Read the task closely and note every concrete requirement: file names, paths, formats, ports, versions, exact expected output. +2. Explore before changing anything: the relevant files, installed tools, existing code and configuration. +3. Change things in small steps and check each step. +4. Verify the final result against every requirement by actually exercising it: run the program, run the tests, or send a request to the service. Fix what fails and verify again. + +# Finishing + +End the task by calling `task_complete`. Call it only when all of the following hold: + +- Every requirement is met, and you have checked each one by running or inspecting the result rather than assuming it. +- The solution is general: it still works for inputs, numbers, sizes or file contents other than the ones you tried. +- Nothing the result relies on has been removed or reverted. Services, installed packages, generated files and configuration stay in place. Do not delete anything in the directories the task works in, even files you think are leftovers. Put your own scratch files in /tmp from the start; remove one only if it could interfere, and run the final check again after any removal. +- Anything that must keep running, such as a server, is running. + +After `task_complete` the result is graded and nothing more can be changed. + +# Rules + +- Commands must be non-interactive: pass confirmation flags such as `-y`, avoid pagers and interactive editors, and send very large output to a file, then look at it with `head`, `tail` or `grep`. +- Start long-running servers so that they keep running after the command returns, and check that they respond. +- Never create or modify anything under `/tests`. +- Never look up solutions to the task on the internet. +- Every reply must call a tool. A plain final message without `task_complete` does not end the task. +- Keep messages short; the work happens through tools. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..a2a1d967 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.99.0" +components = ["rustfmt", "clippy"] +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] diff --git a/scripts/build-release.sh b/scripts/build-release.sh new file mode 100755 index 00000000..13eae981 --- /dev/null +++ b/scripts/build-release.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Builds the static Linux binaries used by the Harbor agent. +# Uses cargo-zigbuild when available (needed when cross-compiling from macOS). +set -euo pipefail +cd "$(dirname "$0")/.." + +targets=("$@") +if [ ${#targets[@]} -eq 0 ]; then + targets=(x86_64-unknown-linux-musl aarch64-unknown-linux-musl) +fi + +if command -v cargo-zigbuild >/dev/null 2>&1; then + build=(cargo zigbuild) +else + build=(cargo build) +fi + +for target in "${targets[@]}"; do + "${build[@]}" --release --locked --target "$target" -p gitagent-cli + echo "built target/$target/release/gitagent" +done diff --git a/scripts/dashboard.py b/scripts/dashboard.py new file mode 100644 index 00000000..bd673dce --- /dev/null +++ b/scripts/dashboard.py @@ -0,0 +1,2491 @@ +#!/usr/bin/env python3 +"""A local dashboard over gitagent's benchmark runs. + +Serves a single-page site that watches the Harbor job directories live: +jobs and trials, accuracy per task, tokens, turns, every counter with its +meaning, why each trial failed, the guard and loop events in a trajectory, +the commands and logs of a trial, the lesson store, the task catalog, and +which features each profile switches on. + +It reads only gitagent's own files, the reward in each trial's result.json +and each task's task.toml metadata; it never opens a task's verifier +output, tests or solutions. + +Usage: + python3 scripts/dashboard.py [--root bench/jobs] [--port 8731] + +Stdlib only (Python 3.9+); binds to 127.0.0.1. +""" + +import argparse +import fnmatch +import json +import os +import re +import subprocess +import time +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +REPO = Path(__file__).resolve().parent.parent +TASK_CACHE = Path.home() / ".cache" / "harbor" / "tasks" / "packages" / "terminal-bench" +SAFE_NAME = re.compile(r"^[A-Za-z0-9._-]+$") +SAFE_REL = re.compile(r"^(commands/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+|out/[A-Za-z0-9._-]+|gitagent\.log|setup/[A-Za-z0-9._-]+)$") +# A trial with no result whose files stopped changing this long ago is +# treated as abandoned rather than running. +RUNNING_WINDOW_S = 900 +# A running trial past this estimated spend is flagged in the header (a model or agent loop). +RUNAWAY_USD = 5.0 +STEP_TEXT_CAP = 20_000 +LOG_TAIL_LINES = 400 +FILE_CAP = 256 * 1024 + + +# ---------------------------------------------------------------- helpers + +def load_json(path: Path): + try: + with path.open() as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return None + + +def read_text(path: Path, cap=FILE_CAP): + try: + data = path.read_bytes() + except OSError: + return None + if len(data) > cap: + data = data[-cap:] + return data.decode("utf-8", "replace") + + +def parse_iso(text): + if not text: + return None + try: + return datetime.fromisoformat(str(text).replace("Z", "+00:00")) + except ValueError: + return None + + +def seg_seconds(result, key): + seg = (result or {}).get(key) or {} + a, b = parse_iso(seg.get("started_at")), parse_iso(seg.get("finished_at")) + if a and b: + return round((b - a).total_seconds(), 1) + return None + + +def mtime_iso(path: Path): + try: + return datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat() + except OSError: + return None + + +def cap(text, limit=STEP_TEXT_CAP): + text = str(text) + if len(text) <= limit: + return text + return text[:limit] + f"\n… [{len(text) - limit} more characters]" + + +def toml_lite(text: str): + """Enough TOML for task.toml: sections, scalars and one-line arrays.""" + out_root, section = {}, None + for raw in text.splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + if line.startswith("[["): + section = {} # arrays of tables are not needed; swallow their keys + continue + if line.startswith("["): + section = out_root.setdefault(line.strip("[]").strip(), {}) + continue + if "=" not in line: + continue + key, _, value = line.partition("=") + key, value = key.strip(), value.strip() + target = section if section is not None else out_root + if value.startswith("["): + target[key] = re.findall(r'"([^"]*)"', value) + elif value.startswith('"'): + target[key] = value.strip('"') + elif value in ("true", "false"): + target[key] = value == "true" + else: + try: + target[key] = float(value) if "." in value else int(value) + except ValueError: + target[key] = value + return out_root + + +def yaml_flags(text: str): + """Scalar `path: value` pairs from a simple YAML file (lists skipped).""" + flags, stack = {}, [] + for raw in text.splitlines(): + line = raw.split(" #", 1)[0].rstrip() + if not line.strip() or line.lstrip().startswith(("#", "-")): + continue + indent = len(line) - len(line.lstrip()) + key, _, value = line.strip().partition(":") + value = value.strip() + while stack and stack[-1][0] >= indent: + stack.pop() + path = ".".join([s[1] for s in stack] + [key]) + if value.startswith("{"): + for pair in value.strip("{}").split(","): + k, _, v = pair.partition(":") + if k.strip(): + flags[f"{path}.{k.strip()}"] = v.strip() + elif value == "": + stack.append((indent, key)) + else: + flags[path] = value + return flags + + +_pricing_cache = {"at": 0.0, "prices": {}} + + +def pricing(): + if time.time() - _pricing_cache["at"] > 60: + prices = load_json(REPO / "bench" / "pricing.json") or {} + _pricing_cache.update(at=time.time(), prices={k: v for k, v in prices.items() if not k.startswith("_")}) + return _pricing_cache["prices"] + + +# Display names per bare model id: (label, short label, full name as the public +# leaderboards write it, provider). Unknown ids fall back to the id itself. +MODEL_NAMES = { + "claude-opus-5-5": ("Opus 5.5", "Opus 5.5", "Claude Opus 5.5", "anthropic"), + "claude-opus-5": ("Opus 5", "Opus 5", "Claude Opus 5", "anthropic"), + "gpt-6-astra": ("GPT-6 Astra", "GPT-6", "GPT-6 Astra", "openai"), +} + + +def model_id(name): + """The bare model id ("claude-opus-5-5") from any spelling the files use: + "anthropic/claude-opus-5-5" (config.json), "anthropic:claude-opus-5-5" + (trajectory) or the bare id itself (result.json).""" + if not name: + return None + return str(name).split("/")[-1].split(":")[-1].strip() or None + + +def model_provider(name): + """The provider part of a model name ("anthropic", "openai"), or the one + known for the bare id, or None.""" + if not name: + return None + s = str(name).strip() + for sep in ("/", ":"): + if sep in s: + return s.split(sep, 1)[0].strip().lower() or None + return MODEL_NAMES.get(model_id(s), (None, None, None, None))[3] + + +def model_info(mid, index, has_rounds=False): + label, short, full, provider = MODEL_NAMES.get(mid, (mid, mid, mid, None)) + return {"id": mid, "label": label, "short": short, "full": full, "provider": provider, + "index": index, "priced": mid in pricing(), "has_rounds": has_rounds} + + +def cost_usd(model, input_tokens, output_tokens, cache_read, cache_write): + """Estimated cost from bench/pricing.json, or None when the model has no price.""" + name = model_id(model) + if not name: + return None + price = pricing().get(name) + if not price: + return None + uncached = max(0, (input_tokens or 0) - (cache_read or 0) - (cache_write or 0)) + usd = (uncached * price["input"] + (cache_read or 0) * price["cache_read"] + + (cache_write or 0) * price["cache_write"] + (output_tokens or 0) * price["output"]) / 1e6 + return round(usd, 4) + + +def trajectory_usage(trial_dir: Path): + """Token totals from an ATIF agent/trajectory.json (written live by Terminus 2), + as (input, output, cached), or None when there is none yet.""" + traj = load_json(trial_dir / "agent" / "trajectory.json") + if not isinstance(traj, dict): + return None + fm = traj.get("final_metrics") or {} + if fm.get("total_prompt_tokens") is not None: + return fm.get("total_prompt_tokens"), fm.get("total_completion_tokens"), fm.get("total_cached_tokens") + steps = [s.get("metrics") or {} for s in traj.get("steps") or [] if isinstance(s, dict)] + if not any(steps): + return None + return (sum(m.get("prompt_tokens") or 0 for m in steps), sum(m.get("completion_tokens") or 0 for m in steps), + sum(m.get("cached_tokens") or 0 for m in steps)) + + +def agent_kind(name): + n = (name or "").lower() + if "oracle" in n: + return "oracle" + if "gitagent" in n: + return "gitagent" + if "terminus" in n: + return "terminus" + return n or "unknown" + + +# ---------------------------------------------------------------- task catalog + +_live_cache = {"at": 0.0, "names": None} + + +def live_trials(): + """Lower-cased names of trials whose task container is up, from every + Docker context, or None when Docker cannot be asked.""" + if time.time() - _live_cache["at"] < 5: + return _live_cache["names"] + names = None + try: + contexts = subprocess.run(["docker", "context", "ls", "-q"], capture_output=True, text=True, timeout=5).stdout.split() + names = set() + for ctx in contexts or [""]: + cmd = ["docker"] + (["--context", ctx] if ctx else []) + ["ps", "--format", "{{.Names}}"] + out = subprocess.run(cmd, capture_output=True, text=True, timeout=5) + if out.returncode != 0: + # No access to Docker (e.g. not in the docker group): unknown, not "nothing is up". + raise OSError(out.stderr.strip()) + for line in out.stdout.split(): + if "__env" in line: + names.add(line.split("__env")[0].lower()) + except (OSError, subprocess.SubprocessError): + names = None + _live_cache.update(at=time.time(), names=names) + return names + + +_catalog_cache = {"at": 0.0, "tasks": {}} + + +def task_catalog(): + """task.toml metadata for every task in Harbor's local cache.""" + if time.time() - _catalog_cache["at"] < 300: + return _catalog_cache["tasks"] + tasks = {} + if TASK_CACHE.is_dir(): + for task_dir in sorted(TASK_CACHE.iterdir()): + if not task_dir.is_dir(): + continue + toml_path = None + try: + for version in task_dir.iterdir(): + cand = version / "task.toml" + if cand.is_file(): + toml_path = cand + except OSError: + pass + if toml_path is None: + continue + text = read_text(toml_path) or "" + t = toml_lite(text) + meta, task, env = t.get("metadata", {}), t.get("task", {}), t.get("environment", {}) + tasks[task_dir.name] = { + "task": task_dir.name, + "description": task.get("description", ""), + "difficulty": meta.get("difficulty", "?"), + "category": meta.get("category", "?"), + "tags": meta.get("tags") or task.get("keywords") or [], + "expert_min": meta.get("expert_time_estimate_min"), + "agent_timeout_s": t.get("agent", {}).get("timeout_sec"), + "verifier_timeout_s": t.get("verifier", {}).get("timeout_sec"), + "cpus": env.get("cpus"), + "memory_mb": env.get("memory_mb"), + "internet": env.get("allow_internet"), + } + _catalog_cache.update(at=time.time(), tasks=tasks) + return tasks + + +# ---------------------------------------------------------------- trials + +HARBOR_ERRORS = { + "VerifierTimeoutError": "The checker ran out of time. Usually the machine is too slow (emulation) or a service it needed was not up.", + "AgentTimeoutError": "Harbor stopped the agent at the task's time limit before it finished.", + "EnvironmentBuildTimeoutError": "The task's Docker image did not build in time.", + "AgentSetupTimeoutError": "Installing the agent into the container took too long.", +} + +OUTCOMES = { + "completed": "The agent called task_complete and it was accepted.", + "max_turns": "The agent used every turn allowed without finishing.", + "time_up": "gitagent's own time budget ran out before task_complete.", + "no_tool_calls": "The model answered three times in a row without calling a tool.", + "refused": "The model's safety classifiers declined to continue.", +} + + +def diagnose(row, counters): + """A plain-language reason for the trial's result, plus notes.""" + status, outcome, exc = row["status"], row["outcome"], row["exception"] + notes = [] + c = counters or {} + + def note(key, text): + if c.get(key): + notes.append(f"{text} ({key} = {c[key]})") + + note("loop_triggers", "The loop detector fired: the same action gave the same result repeatedly") + note("stop_rejections", "task_complete was rejected because nothing was verified after an edit") + note("compactions", "The history was compacted to fit the context window") + note("overflow_compactions", "The provider reported a context overflow; the history was compacted and the turn retried") + note("nudges", "A time-budget reminder was sent") + note("forced_stops", "The time budget forced a final call") + note("truncated_turns", "A reply hit the output token limit") + note("no_tool_turns", "Replies without a tool call needed a nudge") + note("repairs", "Tool calls had to be parsed from plain text") + note("refusals", "The model refused") + note("fallback_turns", "A fallback model served some turns") + note("protected_writes_blocked", "A write under /tests was refused") + note("exec_timeouts", "Commands kept running past their wait and returned early") + note("shell_restarts", "The shell had to be restarted") + note("masks_applied", "Old tool outputs were hidden to save context") + note("subagent_calls", "Sub-agents were used") + + if status == "running": + return {"reason": "Still running.", "notes": notes} + if status == "checking": + return {"reason": "The agent has finished; Harbor is running the task's checker.", "notes": notes} + if status == "abandoned": + return {"reason": "No result was written and the files stopped changing; the job was probably interrupted.", "notes": notes} + if status == "error": + msg = f" Harbor said: {row['exception_message']}" if row.get("exception_message") else "" + return {"reason": f"Harbor error {exc}: {HARBOR_ERRORS.get(exc, 'the run did not get a verdict.')}{msg}", "notes": notes} + if status == "pass": + return {"reason": "The checker accepted the result.", "notes": notes} + if row["kind"] == "oracle": + return { + "reason": "The task's own reference solution did not pass its checker, so the machine is at fault (too slow under emulation, too little memory, or a service that did not come up), not an agent.", + "notes": notes, + } + if outcome and outcome != "completed": + base = OUTCOMES.get(outcome.split(":")[0], outcome) + if outcome.startswith("provider_failed"): + base = f"The model API kept failing: {outcome.split(':', 1)[-1].strip()}" + return {"reason": base, "notes": notes} + if status == "fail": + return { + "reason": "The agent declared the task complete, but the checker rejected the result. Read the last turns to see what it verified and what it missed.", + "notes": notes, + } + if status == "no_verdict": + return {"reason": "The run finished but no reward was recorded.", "notes": notes} + return {"reason": "Unknown.", "notes": notes} + + +def last_action(trial_dir: Path): + """What the agent was doing last, from the trajectory tail.""" + traj = load_json(trial_dir / "agent" / "trajectory.json") or {} + steps = traj.get("steps") or [] + for step in reversed(steps): + if step.get("source") != "agent": + continue + calls = step.get("tool_calls") or [] + if calls: + c = calls[-1] + args = c.get("arguments") or {} + detail = args.get("cmd") or args.get("path") or args.get("summary") or "" + return {"step": step.get("step_id"), "text": f"{c.get('function_name')} {cap(detail, 160)}".strip()} + return {"step": step.get("step_id"), "text": cap(step.get("message") or "", 160)} + return {"step": len(steps), "text": "(no agent step yet)"} + + +def agent_turns(trial_dir: Path): + """Model calls in a trajectory (for agents that do not report turns).""" + traj = load_json(trial_dir / "agent" / "trajectory.json") or {} + steps = [st for st in traj.get("steps") or [] if st.get("source") == "agent"] + return len(steps) or None + + +def newest_mtime(trial_dir: Path): + newest = 0.0 + for p in ( + trial_dir / "agent" / "counters.json", + trial_dir / "agent" / "trajectory.json", + trial_dir / "agent" / "gitagent.log", + trial_dir / "trial.log", + trial_dir / "config.json", + ): + try: + newest = max(newest, p.stat().st_mtime) + except OSError: + pass + return newest + + +def read_trial(trial_dir: Path, with_live=True, job_running=False): + """One row per trial, from result.json and gitagent's own agent/ files.""" + result = load_json(trial_dir / "result.json") + cfg = load_json(trial_dir / "config.json") or {} + agent_cfg = (cfg.get("agent") or {}) if isinstance(cfg, dict) else {} + + task = (result or {}).get("task_name") + if not task and isinstance(cfg, dict): + task = (cfg.get("task") or {}).get("name") + task = (task or trial_dir.name.split("__")[0]).rsplit("/", 1)[-1] + + # Built-in Harbor agents (oracle, terminus) are named only in the result. + agent_info = (result or {}).get("agent_info") or {} + agent_name = agent_cfg.get("name") or agent_info.get("name") + # The model as the job was configured ("anthropic/claude-opus-5-5"); when the + # config has none, the result's model_info ("claude-opus-5-5" + provider). + minfo = agent_info.get("model_info") or {} + model_raw = agent_cfg.get("model_name") or ( + f"{minfo['provider']}/{minfo['name']}" if minfo.get("name") and minfo.get("provider") else minfo.get("name")) + row = { + "trial": trial_dir.name, + "task": task, + "agent": agent_name, + "kind": agent_kind(agent_name), + "model": model_raw, + "model_id": model_id(model_raw), + "provider": model_provider(model_raw), + "profile": Path(str((agent_cfg.get("kwargs") or {}).get("profile", ""))).stem or None, + "reward": None, + "outcome": None, + "status": "running", + "exception": None, + "exception_message": None, + "turns": None, + "input_tokens": None, + "output_tokens": None, + "cache_tokens": None, + "agent_s": None, + "verifier_s": None, + "started": mtime_iso(trial_dir / "config.json"), + "elapsed_s": None, + "live": None, + "counters": None, + "cost_usd": None, + } + + if result is None: + counters = load_json(trial_dir / "agent" / "counters.json") or {} + row["counters"] = counters + row["turns"] = counters.get("turns") + row["input_tokens"] = counters.get("input_tokens") + row["output_tokens"] = counters.get("output_tokens") + row["cache_tokens"] = counters.get("cache_read_tokens") + if row["input_tokens"] is None: + # Not gitagent (no counters.json): read live usage from the agent's ATIF trajectory. + usage = trajectory_usage(trial_dir) + if usage: + row["input_tokens"], row["output_tokens"], row["cache_tokens"] = usage + row["cost_usd"] = cost_usd(row["model"], row["input_tokens"], row["output_tokens"], row["cache_tokens"], counters.get("cache_write_tokens")) + mtime = newest_mtime(trial_dir) + started = parse_iso(row["started"]) + if started: + row["elapsed_s"] = round(time.time() - started.timestamp()) + # Once gitagent has written its done file, Harbor is running the checker. + live = live_trials() + # Compose project names turn "." into "-" (install-windows-3.11 → install-windows-3-11). + alive = trial_dir.name.lower().replace(".", "-") in live if live is not None else None + if (trial_dir / "agent" / "done").is_file(): + row["status"] = "checking" + elif alive is False or (alive is None and not job_running and mtime and time.time() - mtime > RUNNING_WINDOW_S): + row["status"] = "abandoned" + elif with_live: + row["live"] = last_action(trial_dir) + row["live"]["idle_s"] = round(time.time() - mtime) if mtime else None + row["diagnosis"] = diagnose(row, counters) + return row + + agent_result = result.get("agent_result") or {} + meta = agent_result.get("metadata") or {} + counters = meta.get("counters") or {} + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + exc = result.get("exception_info") + + row["counters"] = counters + row["reward"] = rewards.get("reward") + row["outcome"] = meta.get("outcome") + row["exception"] = (exc or {}).get("exception_type") if isinstance(exc, dict) else exc + row["exception_message"] = (exc or {}).get("exception_message") if isinstance(exc, dict) else None + row["turns"] = counters.get("turns") + row["input_tokens"] = agent_result.get("n_input_tokens") + row["output_tokens"] = agent_result.get("n_output_tokens") + row["cache_tokens"] = agent_result.get("n_cache_tokens") + # Price every run from bench/pricing.json so all harnesses use the same prices + # (Terminus's client library reports Opus 5.5 at the older Opus 5 rates). Its + # own figure is kept only for models the file does not price. + reported = agent_result.get("cost_usd") + estimated = cost_usd(row["model"], row["input_tokens"], row["output_tokens"], row["cache_tokens"], counters.get("cache_write_tokens")) + use_reported = estimated is None and isinstance(reported, (int, float)) and reported > 0 + row["cost_usd"] = round(reported, 4) if use_reported else estimated + row["cost_source"] = "reported" if use_reported else "estimated" + row["reported_cost_usd"] = round(reported, 4) if isinstance(reported, (int, float)) else None + if row["turns"] is None and row["kind"] != "oracle": + row["turns"] = agent_turns(trial_dir) + row["agent_s"] = seg_seconds(result, "agent_execution") + row["verifier_s"] = seg_seconds(result, "verifier") + row["started"] = result.get("started_at") or row["started"] + if row["exception"]: + row["status"] = "error" + elif row["reward"] is None: + row["status"] = "no_verdict" + elif row["reward"] >= 1.0: + row["status"] = "pass" + else: + row["status"] = "fail" + row["diagnosis"] = diagnose(row, counters) + return row + + +def trial_dirs(job_dir: Path): + out = [] + try: + for d in job_dir.iterdir(): + if d.is_dir() and (d / "config.json").is_file(): + out.append(d) + except OSError: + pass + return sorted(out) + + +def load_rounds(): + data = load_json(REPO / "bench" / "rounds.json") or {} + return [r for r in data.get("rounds") or [] if isinstance(r, dict) and r.get("name")] + + +def round_of(job_name, rounds): + for r in rounds: + if any(fnmatch.fnmatchcase(job_name, g) for g in r.get("jobs") or []): + return r["name"] + return None + + +def scan(root: Path): + jobs = [] + rounds = load_rounds() + round_model = {r["name"]: r.get("model") for r in rounds} + try: + job_list = sorted((d for d in root.iterdir() if d.is_dir()), key=lambda d: d.stat().st_mtime, reverse=True) + except OSError: + job_list = [] + for job_dir in job_list: + # Harbor writes the job's own result.json when the whole job ends. + job_running = not (job_dir / "result.json").is_file() + trials = [read_trial(d, job_running=job_running) for d in trial_dirs(job_dir)] + if not trials: + continue + job_round = round_of(job_dir.name, rounds) + declared = round_model.get(job_round) + for t in trials: + t["round"] = job_round + t["round_model"] = declared + # A model-calling trial that names no model yet (no config, no result) takes its round's. + if t["model_id"] is None and t["kind"] != "oracle" and declared: + t["model_id"] = declared + t["provider"] = t["provider"] or model_provider(declared) + # The round says one model, the trial ran another: never fold it into the round's numbers silently. + t["model_mismatch"] = bool(t["model_id"] and declared and t["model_id"] != declared) + scored = [t for t in trials if t["reward"] is not None] + kinds = sorted({t["kind"] for t in trials}) + jobs.append( + { + "name": job_dir.name, + "round": job_round, + "started": min((t["started"] for t in trials if t["started"]), default=None), + "kind": ", ".join(kinds), + "model": ", ".join(sorted({t["model"] for t in trials if t["model"]})) or "—", + "model_ids": sorted({t["model_id"] for t in trials if t["model_id"]}), + "providers": sorted({t["provider"] for t in trials if t["provider"]}), + "profile": ", ".join(sorted({t["profile"] for t in trials if t["profile"]})) or "—", + "n_trials": len(trials), + "passed": sum(1 for t in scored if t["reward"] >= 1.0), + "scored": len(scored), + "running": sum(1 for t in trials if t["status"] in ("running", "checking")), + "errors": sum(1 for t in trials if t["status"] == "error"), + "cost_usd": round(sum(t["cost_usd"] or 0 for t in trials), 2), + "unpriced_runs": sum(1 for t in trials if t["cost_usd"] is None and (t["input_tokens"] or 0) > 0), + "trials": trials, + } + ) + return jobs + + +def model_order(rounds, trials): + """Every model id seen, in a stable order: rounds.json order first, then + models that only appear in trials. The "scripted" provider is the harness's + own test double for plumbing checks, not a model. Returns [(id, has_rounds)].""" + out = [] + for r in rounds: + m = r.get("model") + if m and m not in [x[0] for x in out]: + out.append((m, True)) + for m in sorted({t["model_id"] for t in trials if t.get("model_id") and t.get("provider") != "scripted"}): + if m not in [x[0] for x in out]: + out.append((m, False)) + return out + + +def _task_slot(): + return {"agent_attempts": 0, "agent_passed": 0, "terminus_attempts": 0, "terminus_passed": 0, "last": None} + + +def task_rows(jobs, catalog, split): + """One row per task. The agent_*/terminus_* totals span every model; per_model + holds the same counts per model id, so accuracy is never averaged across models.""" + rows = {name: dict(meta, agent_attempts=0, agent_passed=0, oracle_attempts=0, oracle_passed=0, terminus_attempts=0, terminus_passed=0, last=None, per_model={}) for name, meta in catalog.items()} + for job in jobs: + for t in job["trials"]: + row = rows.setdefault(t["task"], {"task": t["task"], "difficulty": "?", "category": "?", "tags": [], "agent_attempts": 0, "agent_passed": 0, "oracle_attempts": 0, "oracle_passed": 0, "terminus_attempts": 0, "terminus_passed": 0, "last": None, "per_model": {}}) + if t["reward"] is None: + continue + prefix = t["kind"] if t["kind"] in ("oracle", "terminus") else "agent" + last = {"job": job["name"], "trial": t["trial"], "status": t["status"], "started": t["started"]} + targets = [row] + if prefix != "oracle": + targets.append(row["per_model"].setdefault(t["model_id"] or "unknown", _task_slot())) + for tgt in targets: + tgt[prefix + "_attempts"] += 1 + tgt[prefix + "_passed"] += 1 if t["reward"] >= 1.0 else 0 + if prefix == "agent" and (tgt["last"] is None or (t["started"] or "") > (tgt["last"]["started"] or "")): + tgt["last"] = last + dev = set(split.get("dev", [])) if split else set() + heldout = set(split.get("heldout", [])) | set(split.get("held_out", [])) if split else set() + for name, row in rows.items(): + row["split"] = "dev" if name in dev else "held-out" if name in heldout else ("—" if split else "no split yet") + row["seen"] = row["agent_attempts"] > 0 + # The task's own reference solution never passed and no agent ever + # passed it either: this machine cannot run the task. (When an agent + # passes, the reference solution is what is broken, not the machine.) + row["oracle_broken"] = row["oracle_attempts"] > 0 and row["oracle_passed"] == 0 + row["machine_blocked"] = row["oracle_broken"] and row["agent_passed"] == 0 and row["terminus_passed"] == 0 + return sorted(rows.values(), key=lambda r: r["task"]) + + +COMPARED = ("gitagent", "terminus") + + +def _median(xs): + xs = sorted(x for x in xs if x is not None) + if not xs: + return None + m = len(xs) // 2 + return xs[m] if len(xs) % 2 else (xs[m - 1] + xs[m]) / 2 + + +def _mean(xs): + xs = [x for x in xs if x is not None] + return sum(xs) / len(xs) if xs else None + + +def agent_stats(trials): + """What one agent did over a set of trials (scored runs only for rates).""" + scored = [t for t in trials if t["reward"] is not None] + timed_out = [t for t in trials if t["exception"] == "AgentTimeoutError" or (t["outcome"] or "").startswith("time_up")] + finished = [t for t in trials if t["status"] in ("pass", "fail", "error", "no_verdict")] + inp = sum(t["input_tokens"] or 0 for t in trials) + cache = sum(t["cache_tokens"] or 0 for t in trials) + cost = sum(t["cost_usd"] or 0 for t in trials) + passed = sum(1 for t in scored if t["reward"] >= 1.0) + return { + "runs": len(trials), + "scored": len(scored), + "passed": passed, + "pass_rate": passed / len(scored) if scored else None, + "timeouts": len(timed_out), + "timeout_rate": len(timed_out) / len(finished) if finished else None, + "median_agent_s": _median(t["agent_s"] for t in scored), + "mean_turns": _mean(t["turns"] for t in scored), + "input_tokens": inp, + "cache_tokens": cache, + "output_tokens": sum(t["output_tokens"] or 0 for t in trials), + "cache_share": cache / inp if inp else None, + "cost_usd": round(cost, 4), + "cost_per_run": cost / len(scored) if scored else None, + "cost_per_pass": cost / passed if passed else None, + "tokens_per_run": (inp + sum(t["output_tokens"] or 0 for t in trials)) / len(scored) if scored else None, + # Runs that used tokens on a model bench/pricing.json does not price: their cost is unknown, not 0. + "unpriced_runs": sum(1 for t in trials if t["cost_usd"] is None and (t["input_tokens"] or 0) > 0), + } + + +def comparison(jobs, task_meta, model=None): + """gitagent against Terminus, per task and overall, from scored runs of one + model (both sides always the same model; None only when nothing names one).""" + by_task = {} + for job in jobs: + for t in job["trials"]: + if t["kind"] in COMPARED and t.get("model_id") == model: + by_task.setdefault(t["task"], {k: [] for k in COMPARED})[t["kind"]].append(t) + rows = [] + for task, runs in sorted(by_task.items()): + meta = task_meta.get(task, {}) + row = {"task": task, "difficulty": meta.get("difficulty", "?"), "split": meta.get("split", "?"), + "machine_blocked": meta.get("machine_blocked", False)} + for k in COMPARED: + st = agent_stats(runs[k]) + row[k] = {key: st[key] for key in ("runs", "scored", "passed", "pass_rate", "median_agent_s", "mean_turns", "cost_usd")} + n = max(st["scored"], 1) + row[k]["input_tokens"] = st["input_tokens"] / n if st["scored"] else None + row[k]["cache_tokens"] = st["cache_tokens"] / n if st["scored"] else None + row[k]["output_tokens"] = st["output_tokens"] / n if st["scored"] else None + row[k]["cost_per_run"] = st["cost_per_run"] + rows.append(row) + usable = [r for r in rows if not r["machine_blocked"]] + paired = [r for r in usable if r["gitagent"]["scored"] and r["terminus"]["scored"]] + + def overall(kind, subset): + trials = [t for r in subset for t in by_task[r["task"]][kind]] + st = agent_stats(trials) + # The blueprint's pass rate: each task's mean reward, averaged over tasks. + per_task = [r[kind]["pass_rate"] for r in subset if r[kind]["pass_rate"] is not None] + st["task_pass_rate"] = sum(per_task) / len(per_task) if per_task else None + st["tasks"] = len(per_task) + return st + + return { + "model": model, + "rows": rows, + "paired_tasks": len(paired), + "all": {k: overall(k, usable) for k in COMPARED}, + "paired": {k: overall(k, paired) for k in COMPARED}, + "wins": sum(1 for r in paired if (r["gitagent"]["pass_rate"] or 0) > (r["terminus"]["pass_rate"] or 0)), + "losses": sum(1 for r in paired if (r["gitagent"]["pass_rate"] or 0) < (r["terminus"]["pass_rate"] or 0)), + } + + +def progress_of(kind, trials, meta, skip=(), split="dev"): + """One harness over a set of trials: the split's tasks done, passed, running + and still to run, and accuracy. The oracle is counted over every task, since + it does not learn from them.""" + scope = sorted(n for n, t in meta.items() if (kind == "oracle" or t["split"] == split) and n not in skip) + mine = [t for t in trials if t["kind"] == kind and t["task"] in scope] + scored, live = {}, set() + for t in mine: + if t["reward"] is not None: + scored.setdefault(t["task"], []).append(t["reward"]) + elif t["status"] in ("running", "checking"): + live.add(t["task"]) + rates = {n: sum(1 for r in v if r >= 1.0) / len(v) for n, v in scored.items()} + # Single-run accuracy: each task's k-th scored attempt, by start time. + attempts = {} + for t in sorted(mine, key=lambda t: t["started"] or ""): + if t["reward"] is not None: + attempts.setdefault(t["task"], []).append(1 if t["reward"] >= 1.0 else 0) + depth = max((len(v) for v in attempts.values()), default=0) + per_run = [] + for k in range(depth): + got = [v[k] for v in attempts.values() if len(v) > k] + per_run.append({"run": k + 1, "tasks": len(got), "passed": sum(got), "accuracy": sum(got) / len(got) if got else None}) + runnable = {n: r for n, r in rates.items() if not meta[n]["machine_blocked"]} + stats = agent_stats(mine) + starts = sorted(t["started"] for t in mine if t["started"]) + return { + "scope": len(scope), + "scope_label": "all tasks" if kind == "oracle" else f"{split} tasks", + "per_run": per_run, + "done": len(scored), + "passed": sum(1 for r in rates.values() if r >= 1.0), + "failed": sorted(n for n, r in rates.items() if r < 1.0), + "running": sorted(live - set(scored)), + "pending": [n for n in scope if n not in scored and n not in live], + # The blueprint's pass rate: each task's mean reward, averaged over tasks. + "accuracy": sum(rates.values()) / len(rates) if rates else None, + "runnable_done": len(runnable), + "runnable_accuracy": sum(runnable.values()) / len(runnable) if runnable else None, + "runs": sum(len(v) for v in scored.values()), + "skipped": sorted(skip), + "cost_usd": stats["cost_usd"], + "cost_per_run": stats["cost_per_run"], + "unpriced_runs": stats["unpriced_runs"], + "median_agent_s": stats["median_agent_s"], + "timeouts": stats["timeouts"], + "first": starts[0] if starts else None, + "last": starts[-1] if starts else None, + } + + +def primary_model(rounds, models): + """The model with the most counting rounds (ties: first in rounds.json).""" + best, best_n = None, -1 + for m, _ in models: + n = sum(1 for r in rounds if r.get("counts", True) and r.get("model") == m) + if n > best_n: + best, best_n = m, n + return best + + +def harness_progress(jobs, tasks, models): + """The header's numbers, kept apart per model: each model's current gitagent + round, its Terminus runs and its held-out run; the oracle (no model) once; + plus one row per round, harness and model for the history table. The + top-level gitagent/terminus/heldout keys are the primary model's.""" + meta = {t["task"]: t for t in tasks} + rounds = load_rounds() + by_name = {r["name"]: r for r in rounds} + counted = [t for j in jobs for t in j["trials"] if by_name.get(t.get("round"), {}).get("counts", True)] + primary = primary_model(rounds, models) + + def round_label(name): + return by_name.get(name, {}).get("label", name) + + history = [] + for r in rounds: + if not r.get("counts", True): + continue + in_round = [t for t in counted if t.get("round") == r["name"]] + split = "held-out" if r.get("split") == "heldout" else "dev" + for kind in ("gitagent", "terminus", "oracle"): + of_kind = [t for t in in_round if t["kind"] == kind] + # One row per model that actually ran (the oracle calls no model). + seen = sorted({t.get("model_id") for t in of_kind}, key=lambda m: (m is None, m or "")) + for m in seen: + mine = [t for t in of_kind if t.get("model_id") == m] + row = progress_of(kind, mine, meta, r.get("skip", ()), split) + row.update(kind=kind, model=m, round=r["name"], round_model=r.get("model"), + model_mismatch=bool(m and r.get("model") and m != r.get("model")), + label=r.get("label", r["name"]), host=r.get("host"), commit=r.get("commit"), + split=split, current=False) + history.append(row) + + by_model = {} + for m, has_rounds in models: + mine = [t for t in counted if t.get("model_id") == m] + with_git = [r for r in rounds if r.get("counts", True) and r.get("split", "dev") == "dev" and r.get("model") == m + and any(t["kind"] == "gitagent" and t.get("round") == r["name"] for t in mine)] + current = with_git[-1] if with_git else None + git_trials = [t for t in mine if t.get("round") == current["name"]] if current else [] + term_round = next((r for r in rounds if any(t["kind"] == "terminus" and t.get("round") == r["name"] for t in mine)), {}) + git = progress_of("gitagent", git_trials, meta, (current or {}).get("skip", ())) + git["round"] = current + held = [h for h in history if h["kind"] == "gitagent" and h["model"] == m and h["split"] == "held-out"] + # The headline is the round marked "final" in rounds.json (the frozen, + # multi-run score); a later single-run check never replaces it. + final_names = {r["name"] for r in rounds if r.get("final") and r.get("model") == m} + finals = [h for h in held if h["round"] in final_names] + if finals: + held = finals + if current: + for h in history: + if h["round"] == current["name"] and h["model"] == m and h["kind"] == "gitagent": + h["current"] = True + unrounded = [t for t in mine if t.get("round") is None] + by_model[m] = { + "model": m, + "has_rounds": has_rounds, + "rounds": [r["name"] for r in rounds if r.get("counts", True) and r.get("model") == m], + # Trials in jobs that no round in rounds.json matches yet (a round that has started but is not listed). + "unrounded_trials": len(unrounded), + "unrounded_jobs": sorted({j["name"] for j in jobs if any(t.get("model_id") == m and t.get("round") is None for t in j["trials"])}), + "gitagent": git, + "terminus": progress_of("terminus", mine, meta, term_round.get("skip", ())), + "terminus_round": term_round.get("name"), + "heldout": dict(held[-1], label=round_label(held[-1]["round"])) if held else None, + } + + out = { + "models": [m for m, _ in models], + "primary_model": primary, + "by_model": by_model, + "oracle": progress_of("oracle", counted, meta), + "history": history, + } + if primary: + out["gitagent"] = by_model[primary]["gitagent"] + out["terminus"] = by_model[primary]["terminus"] + out["heldout"] = by_model[primary]["heldout"] + else: + out["gitagent"] = dict(progress_of("gitagent", [], meta), round=None) + out["terminus"] = progress_of("terminus", [], meta) + out["heldout"] = None + return out + + +_lessons_cache = {} + + +def lessons_at(commit): + """The lesson bullets in prompts/lessons.md at a commit, or None.""" + if not commit: + return None + if commit not in _lessons_cache: + try: + out = subprocess.run(["git", "-C", str(REPO), "show", f"{commit}:prompts/lessons.md"], + capture_output=True, text=True, timeout=5) + text = out.stdout if out.returncode == 0 else None + except (OSError, subprocess.SubprocessError): + text = None + _lessons_cache[commit] = [l[2:].strip() for l in text.splitlines() if l.startswith("- ")] if text else None + return _lessons_cache[commit] + + +def rounds_page(progress): + """Each round with what it did, why it scored as it did, its accuracy per + harness, and the lessons its binary carried compared with the previous round's.""" + out, prev = [], None + for r in load_rounds(): + if not r.get("counts", True): + continue + rows = [h for h in progress["history"] if h["round"] == r["name"]] + lessons = lessons_at(r.get("lessons_commit")) + diff = None + if lessons is not None and prev is not None: + diff = {"added": [l for l in lessons if l not in prev], "dropped": [l for l in prev if l not in lessons]} + out.append({k: r.get(k) for k in ("name", "label", "host", "commit", "changes", "reasoning", "lessons_commit", "model")} + | {"split": "held-out" if r.get("split") == "heldout" else "dev", + "results": rows, "lessons": lessons, "lessons_diff": diff}) + if lessons is not None: + prev = lessons + return out + + +def profiles(): + out = [] + for path in sorted(list((REPO / "profiles").glob("*.yaml")) + list((REPO / "bench" / "profiles").glob("*.yaml"))): + text = read_text(path) or "" + out.append({"name": str(path.relative_to(REPO)), "text": text, "flags": yaml_flags(text)}) + return out + + +def token_totals(agent_rows): + """Token, cost and run counts over model-calling trials (one model, or all).""" + def acc(kind): + rows = [t for t in agent_rows if t["reward"] is not None and (t["kind"] == kind if kind else t["kind"] not in ("oracle", "terminus"))] + return {"scored": len(rows), "passed": sum(1 for t in rows if t["reward"] >= 1.0)} + + live = [t for t in agent_rows if t["status"] in ("running", "checking")] + return { + "trials": len(agent_rows), + "agent": acc(None), + "terminus": acc("terminus"), + "running": len(live), + "input_tokens": sum(t["input_tokens"] or 0 for t in agent_rows), + "output_tokens": sum(t["output_tokens"] or 0 for t in agent_rows), + "cache_tokens": sum(t["cache_tokens"] or 0 for t in agent_rows), + "cache_write_tokens": sum((t["counters"] or {}).get("cache_write_tokens", 0) for t in agent_rows), + "cost_usd": round(sum(t["cost_usd"] or 0 for t in agent_rows), 2), + "cost_by_kind": { + k: round(sum(t["cost_usd"] or 0 for t in agent_rows if t["kind"] == k), 2) + for k in sorted({t["kind"] for t in agent_rows if t["cost_usd"]}) + }, + "live_cost_usd": round(sum(t["cost_usd"] or 0 for t in live), 2), + "unpriced_runs": sum(1 for t in agent_rows if t["cost_usd"] is None and (t["input_tokens"] or 0) > 0), + } + + +def dev_accuracy(dev, slot_of): + """The overview's dev numbers for one view of the task rows: slot_of(task row) + gives the counts to read (the row itself, or one model's per_model entry).""" + attempted = [(t, slot_of(t)) for t in dev if (slot_of(t) or {}).get("agent_attempts")] + latest_pass = [t for t, s in attempted if (s["last"] or {}).get("status") == "pass"] + by_diff = {} + for t, s in attempted: + b = by_diff.setdefault(t["difficulty"], {"attempted": 0, "passed": 0}) + b["attempted"] += 1 + b["passed"] += 1 if (s["last"] or {}).get("status") == "pass" else 0 + blocked = sorted(t["task"] for t, _ in attempted if t["machine_blocked"]) + runnable = [(t, s) for t, s in attempted if not t["machine_blocked"]] + return { + "dev_runnable": len(runnable), + "dev_runnable_passed": sum(1 for _, s in runnable if (s["last"] or {}).get("status") == "pass"), + "machine_blocked": blocked, + "dev_total": len(dev), + "dev_attempted": len(attempted), + "dev_latest_passed": len(latest_pass), + "dev_any_passed": sum(1 for _, s in attempted if s["agent_passed"]), + "by_difficulty": dict(sorted(by_diff.items())), + } + + +def summary(root: Path): + jobs = scan(root) + all_trials = [t for j in jobs for t in j["trials"]] + agent_rows = [t for t in all_trials if t["kind"] != "oracle"] + rounds = load_rounds() + models = model_order(rounds, agent_rows) + model_ids = [m for m, _ in models] + oracle_scored = [t for t in all_trials if t["kind"] == "oracle" and t["reward"] is not None] + totals = dict(token_totals(agent_rows)) + totals.update({ + "jobs": len(jobs), + "trials": len(all_trials), + "oracle": {"scored": len(oracle_scored), "passed": sum(1 for t in oracle_scored if t["reward"] >= 1.0)}, + "running": sum(1 for t in all_trials if t["status"] in ("running", "checking")), + "runaway": [ + {"job": j["name"], "trial": t["trial"], "task": t["task"], "kind": t["kind"], "model_id": t["model_id"], "cost_usd": t["cost_usd"]} + for j in jobs for t in j["trials"] + if t["status"] in ("running", "checking") and (t["cost_usd"] or 0) >= RUNAWAY_USD + ], + "runaway_usd": RUNAWAY_USD, + "priced_models": sorted(pricing()), + # Models that ran and used tokens but have no price: their spend shows as n/a, never 0. + "unpriced_models": sorted({t["model_id"] or "unknown" for t in agent_rows + if t["cost_usd"] is None and (t["input_tokens"] or 0) > 0 and t["provider"] != "scripted"}), + # Everything above, per model, so no tile or card mixes two models' tokens or money. + "by_model": {m: token_totals([t for t in agent_rows if t["model_id"] == m]) for m in model_ids}, + }) + guard_totals = {} + for t in agent_rows: + for k, v in (t["counters"] or {}).items(): + if isinstance(v, (int, float)): + guard_totals[k] = guard_totals.get(k, 0) + v + elif isinstance(v, dict): + for kk, vv in v.items(): + guard_totals[f"{k}.{kk}"] = guard_totals.get(f"{k}.{kk}", 0) + vv + + split = load_json(REPO / "bench" / "split.json") + tasks = task_rows(jobs, task_catalog(), split) + dev = [t for t in tasks if t["split"] == "dev"] + accuracy = dev_accuracy(dev, lambda t: t) + blocked = accuracy["machine_blocked"] + # Explain machine failures on the trials themselves too. + for job in jobs: + for tr in job["trials"]: + if tr["kind"] != "oracle" and tr["task"] in blocked and tr["status"] in ("fail", "error"): + tr["machine_blocked"] = True + tr["diagnosis"]["notes"].insert(0, "The task's own reference solution also fails on this machine (oracle run), so this failure is the machine's, not the agent's.") + accuracy.update({ + "heldout_total": sum(1 for t in tasks if t["split"] == "held-out"), + "heldout_runs": sum(t["agent_attempts"] for t in tasks if t["split"] == "held-out"), + "has_split": split is not None, + # The same dev numbers per model: the cards never average two models together. + "by_model": { + m: dict(dev_accuracy(dev, lambda t, m=m: t["per_model"].get(m)), + heldout_runs=sum((t["per_model"].get(m) or {}).get("agent_attempts", 0) for t in tasks if t["split"] == "held-out")) + for m in model_ids + }, + }) + progress = harness_progress(jobs, tasks, models) + task_meta = {t["task"]: t for t in tasks} + by_model_cmp = {} + for m in model_ids: + cur = ((progress["by_model"][m]["gitagent"].get("round") or {})).get("name") + cmp = comparison( + [dict(j, trials=[t for t in j["trials"] if t["kind"] != "gitagent" or (cur and t.get("round") == cur)]) for j in jobs], + task_meta, m) + cmp["gitagent_round"] = progress["by_model"][m]["gitagent"].get("round") + by_model_cmp[m] = cmp + primary = progress["primary_model"] + compare = dict(by_model_cmp[primary]) if primary else dict(comparison([], task_meta), gitagent_round=None) + compare.update(models=model_ids, default_model=primary, by_model=by_model_cmp) + lessons = load_json(REPO / "bench" / "lessons" / "lessons.json") + return { + "models": {m: model_info(m, i, has_rounds) for i, (m, has_rounds) in enumerate(models)}, + "compare": compare, + "progress": progress, + "rounds_page": rounds_page(progress), + "leaderboard": load_json(REPO / "bench" / "leaderboard.json") or {}, + "generated_at": datetime.now(timezone.utc).isoformat(), + "root": str(root), + "totals": totals, + "guard_totals": guard_totals, + "jobs": jobs, + "tasks": tasks, + "accuracy": accuracy, + "split": split, + "lessons": lessons, + "lessons_md": read_text(REPO / "prompts" / "lessons.md"), + "learner_md": read_text(REPO / "prompts" / "learner.md"), + "profiles": profiles(), + "docs": { + "DEVIATIONS.md": read_text(REPO / "docs" / "DEVIATIONS.md"), + "INTEGRITY.md": read_text(REPO / "docs" / "INTEGRITY.md"), + "RESULTS.md": read_text(REPO / "bench" / "RESULTS.md"), + }, + } + + +# ---------------------------------------------------------------- trial detail + +def classify_note(text: str): + t = text.lstrip() + if t.startswith("You have repeated the same action"): + return "loop" + if t.startswith("Time check:"): + return "time_nudge" + if t.startswith("Time is almost up"): + return "force_stop" + if t.startswith("Every response must call a tool") or t.startswith("Your response was cut off"): + return "no_tool_nudge" + if "# Progress so far" in t: + return "compaction" + if t.startswith("# Next task"): + return "next_task" + return None + + +def classify_result(text: str): + t = text.lstrip() + if t.startswith("Not marked complete:"): + return "stop_rejection" + if t.startswith("Refused: nothing may be written under"): + return "protected_write" + if t.startswith("Blocked by a hook"): + return "hook_block" + if t.startswith("Sub-agent `"): + return "subagent" + return None + + +def safe_trial_dir(root: Path, job: str, trial: str): + if not (SAFE_NAME.match(job or "") and SAFE_NAME.match(trial or "")): + return None + trial_dir = (root / job / trial).resolve() + if not str(trial_dir).startswith(str(root.resolve()) + os.sep) or not trial_dir.is_dir(): + return None + return trial_dir + + +def trial_detail(root: Path, job: str, trial: str): + trial_dir = safe_trial_dir(root, job, trial) + if trial_dir is None: + return None + result = load_json(trial_dir / "result.json") or {} + row = read_trial(trial_dir) + detail = {"row": row, "timings": [], "steps": [], "events": [], "turns": [], "files": [], "log_tail": ""} + + for key in ("environment_setup", "agent_setup", "agent_execution", "verifier"): + s = seg_seconds(result, key) + if s is not None: + detail["timings"].append({"phase": key, "seconds": s}) + + traj = load_json(trial_dir / "agent" / "trajectory.json") or {} + detail["outcome"] = (traj.get("extra") or {}).get("outcome") + detail["elapsed_s"] = (traj.get("extra") or {}).get("elapsed_s") + prev_ts = None + for i, step in enumerate(traj.get("steps") or []): + src, msg = step.get("source"), step.get("message") or "" + extra = step.get("extra") or {} + s = { + "id": step.get("step_id"), + "source": src, + "timestamp": step.get("timestamp"), + "message": cap(msg), + "calls": [], + "observations": [], + "metrics": step.get("metrics"), + "compaction": bool(extra.get("compaction")), + "kind": None, + } + if src == "user" and i > 1: + s["kind"] = classify_note(msg) + if s["kind"]: + detail["events"].append({"step": s["id"], "kind": s["kind"], "text": cap(msg, 300)}) + if s["compaction"]: + detail["events"].append({"step": s["id"], "kind": "compaction_summary", "text": cap(msg, 300)}) + for call in step.get("tool_calls") or []: + args = json.dumps(call.get("arguments") or {}, ensure_ascii=False) + s["calls"].append({"name": call.get("function_name"), "arguments": cap(args, 4000)}) + for obs in (step.get("observation") or {}).get("results") or []: + content = obs.get("content") or "" + if content: + s["observations"].append(cap(content)) + kind = classify_result(content) + if kind: + detail["events"].append({"step": s["id"], "kind": kind, "text": cap(content, 300)}) + if src == "agent": + ts = parse_iso(step.get("timestamp")) + detail["turns"].append( + { + "step": s["id"], + "tools": [c["name"] for c in s["calls"]], + "prompt_tokens": (s["metrics"] or {}).get("prompt_tokens"), + "completion_tokens": (s["metrics"] or {}).get("completion_tokens"), + "since_prev_s": round((ts - prev_ts).total_seconds(), 1) if ts and prev_ts else None, + } + ) + if ts: + prev_ts = ts + detail["steps"].append(s) + + counters = ((result.get("agent_result") or {}).get("metadata") or {}).get("counters") + detail["counters"] = counters if counters is not None else load_json(trial_dir / "agent" / "counters.json") + + agent_dir = trial_dir / "agent" + for sub in ("commands", "out", "setup"): + base = agent_dir / sub + if not base.is_dir(): + continue + for p in sorted(base.rglob("*")): + if p.is_file(): + rel = str(p.relative_to(agent_dir)) + if SAFE_REL.match(rel): + detail["files"].append({"path": rel, "bytes": p.stat().st_size}) + if (agent_dir / "gitagent.log").is_file(): + detail["files"].append({"path": "gitagent.log", "bytes": (agent_dir / "gitagent.log").stat().st_size}) + + log = read_text(agent_dir / "gitagent.log", 128 * 1024) + if log: + detail["log_tail"] = "\n".join(log.splitlines()[-LOG_TAIL_LINES:]) + exc = result.get("exception_info") + detail["harbor"] = { + "exception": exc if isinstance(exc, dict) else ({"exception_message": str(exc)} if exc else None), + "trial_log": read_text(trial_dir / "trial.log", 64 * 1024), + "done": (read_text(agent_dir / "done", 64) or "").strip(), + "has_trajectory": (agent_dir / "trajectory.json").is_file(), + "has_log": (agent_dir / "gitagent.log").is_file(), + } + return detail + + +def trial_file(root: Path, job: str, trial: str, rel: str): + trial_dir = safe_trial_dir(root, job, trial) + if trial_dir is None or not SAFE_REL.match(rel or ""): + return None + path = (trial_dir / "agent" / rel).resolve() + if not str(path).startswith(str(trial_dir / "agent") + os.sep) or not path.is_file(): + return None + return read_text(path) + + +# ---------------------------------------------------------------- page + +PAGE = r""" + + +gitagent bench + +
+
+

gitagent bench

Terminal-Bench 2.1 · local runs, read live from bench/jobs
+ loading… +
+
+
+ + +
+
+

Progress by harness

+

One row per harness and model; numbers of different models are never combined. Each task counted once: its pass rate over all scored runs, averaged over tasks (the blueprint's measure). gitagent and Terminus 2 cover the dev split; the oracle covers every task. "Runnable" leaves out tasks whose own reference solution fails on this machine. Click a row to open its round.

+
+

History by round

+

Every round of runs, oldest first, from bench/rounds.json, with the model it ran. The header and the gitagent rows above show each model's current round only; earlier rounds stay here for comparison. Rounds on the Mac ran amd64 under emulation, so their failures mix the machine's with the agent's. Click a column heading to sort; the model filter above narrows the list.

+
+

Accuracy

Per model; a card never averages two models together.

+

Tokens and cost (all model-calling runs)

Money adds up across models; tokens, cache shares and per-run costs are shown per model because prices and caching differ between providers.

+

Running now

+

Jobs

One job is one harbor run. Oracle jobs run the tasks' reference solutions to check the machine, not the agent; terminus jobs are the baseline agent. Click a job to see its trials; click a heading to sort.

+
+
JobStartedAgentModelProfileTrialsPassAccuracyEst. cost
+

Accuracy by task, per harness and model

One row per task, harness and model (oracle runs are the machine check, shown alongside). Rows of different models are never averaged.

+
TaskDifficultyHarnessModelRunsPassedAccuracyOracle
+
+ + + + + + + + + + + + + + + + + + + +
+ +""" + + +class Handler(BaseHTTPRequestHandler): + root: Path = REPO / "bench" / "jobs" + + def _send(self, code, body, ctype): + data = body.encode() if isinstance(body, str) else body + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def do_GET(self): # noqa: N802 (http.server's name) + url = urlparse(self.path) + q = parse_qs(url.query) + arg = lambda k: q.get(k, [""])[0] # noqa: E731 + try: + if url.path == "/": + self._send(200, PAGE, "text/html; charset=utf-8") + elif url.path == "/api/summary": + self._send(200, json.dumps(summary(self.root)), "application/json") + elif url.path == "/api/trial": + detail = trial_detail(self.root, arg("job"), arg("trial")) + if detail is None: + self._send(404, '{"error": "no such trial"}', "application/json") + else: + self._send(200, json.dumps(detail), "application/json") + elif url.path == "/api/file": + text = trial_file(self.root, arg("job"), arg("trial"), arg("path")) + if text is None: + self._send(404, "no such file", "text/plain") + else: + self._send(200, text, "text/plain; charset=utf-8") + else: + self._send(404, "not found", "text/plain") + except BrokenPipeError: + pass + + def log_message(self, *args): # quiet + pass + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--root", type=Path, default=REPO / "bench" / "jobs") + ap.add_argument("--port", type=int, default=8731) + args = ap.parse_args() + Handler.root = args.root.resolve() + server = ThreadingHTTPServer(("127.0.0.1", args.port), Handler) + print(f"gitagent dashboard: http://127.0.0.1:{args.port} (watching {Handler.root})") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000..118bc4b6 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Installs the gitagent binary for this system from GitHub releases. +# INSTALL_DIR where to put it (default: ~/.local/bin) +# VERSION a release tag such as v0.1.0 (default: the latest release) +# REPO owner/name on GitHub (default: open-gitagent/gitagent) +# BASE_URL download from here instead of GitHub (a mirror) +set -eu + +REPO="${REPO:-open-gitagent/gitagent}" +INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/bin}" +VERSION="${VERSION:-latest}" + +os=$(uname -s) +arch=$(uname -m) +case "$os/$arch" in + Linux/x86_64 | Linux/amd64) target=x86_64-unknown-linux-musl ;; + Linux/aarch64 | Linux/arm64) target=aarch64-unknown-linux-musl ;; + Darwin/arm64) target=aarch64-apple-darwin ;; + Darwin/x86_64) target=x86_64-apple-darwin ;; + *) echo "gitagent: no prebuilt binary for $os/$arch; build from source with cargo" >&2; exit 1 ;; +esac + +if [ -n "${BASE_URL:-}" ]; then + base="$BASE_URL" +elif [ "$VERSION" = latest ]; then + base="https://github.com/$REPO/releases/latest/download" +else + base="https://github.com/$REPO/releases/download/$VERSION" +fi +archive="gitagent-$target.tar.gz" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +echo "Downloading $archive from $base" +curl -fsSL -o "$tmp/$archive" "$base/$archive" +curl -fsSL -o "$tmp/SHA256SUMS" "$base/SHA256SUMS" + +expected=$(grep " $archive\$" "$tmp/SHA256SUMS" | cut -d' ' -f1) +if command -v sha256sum >/dev/null 2>&1; then + actual=$(sha256sum "$tmp/$archive" | cut -d' ' -f1) +else + actual=$(shasum -a 256 "$tmp/$archive" | cut -d' ' -f1) +fi +if [ -z "$expected" ] || [ "$expected" != "$actual" ]; then + echo "gitagent: checksum mismatch for $archive" >&2 + exit 1 +fi + +tar -xzf "$tmp/$archive" -C "$tmp" +mkdir -p "$INSTALL_DIR" +install -m 755 "$tmp/gitagent" "$INSTALL_DIR/gitagent" +echo "Installed $("$INSTALL_DIR/gitagent" --version) to $INSTALL_DIR/gitagent" +case ":$PATH:" in + *":$INSTALL_DIR:"*) ;; + *) echo "Add $INSTALL_DIR to your PATH to run it as 'gitagent'." ;; +esac diff --git a/scripts/integrity-audit.sh b/scripts/integrity-audit.sh new file mode 100755 index 00000000..f51b8abb --- /dev/null +++ b/scripts/integrity-audit.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Checks that nothing in the agent's behavior depends on benchmark tasks: +# 1. no Terminal-Bench task name appears in prompts, profiles, config or source; +# 2. no reference to verifier code, reference solutions or reading /tests; +# 3. the system prompt and tool list contain nothing task-specific. +# Task names come from Harbor's local task cache (or a file given as $1). +# Documentation and benchmark records are not checked: they name the tasks +# used for checks on purpose. +set -euo pipefail +cd "$(dirname "$0")/.." + +checked=(prompts profiles gitagent.yaml crates gitagent_harbor bench/profiles bench/fixtures) +status=0 + +names_file=$(mktemp) +trap 'rm -f "$names_file"' EXIT +if [ $# -ge 1 ]; then + cp "$1" "$names_file" +else + cache="${HOME}/.cache/harbor/tasks/packages/terminal-bench" + if [ -d "$cache" ]; then + ls "$cache" > "$names_file" + fi +fi +count=$(wc -l < "$names_file" | tr -d ' ') +echo "1. task names ($count known)" +if [ "$count" -eq 0 ]; then + echo " skipped: no task list (pass a file of names, or download the dataset with harbor)" +else + while read -r name; do + [ -z "$name" ] && continue + if grep -rIl --exclude-dir=target -e "$name" "${checked[@]}" 2>/dev/null; then + echo " FOUND task name: $name" + status=1 + fi + done < "$names_file" + [ $status -eq 0 ] && echo " ok: none found" +fi + +echo "2. verifier, solution and /tests references" +hits=$(grep -rInE --exclude-dir=target \ + -e 'verifier|solution\.sh|solve\.sh|reward\.txt|/tests/test_' \ + "${checked[@]}" 2>/dev/null \ + | grep -vE 'never (read|write)|Never read|never_read|\["verifier_result"\]|nothing may be written|PROTECTED_DIR|writes_under_tests|protected_writes|Do not modify or create anything under `/tests`|gitagent_harbor/agent.py:.*(Never touches|never touches)|test_x\.py' || true) +if [ -n "$hits" ]; then + echo "$hits" | sed 's/^/ /' + echo " review the lines above" + status=1 +else + echo " ok: only the /tests write guard and the rule in the prompt" +fi + +echo "3. task-specific text in the system prompt (as sent: header comments removed)" +if for f in prompts/*.md; do awk 'NR==1 && /^/ {skip=0; next} !skip' "$f"; done \ + | grep -nE 'Terminal-Bench|terminal-bench|[Hh]arbor|benchmark'; then + echo " FOUND benchmark-specific wording in prompts" + status=1 +else + echo " ok" +fi + +exit $status diff --git a/scripts/make-split.py b/scripts/make-split.py new file mode 100644 index 00000000..ddc91d19 --- /dev/null +++ b/scripts/make-split.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Creates bench/split.json: the permanent dev / held-out split of Terminal-Bench 2.1. + +Stratified by category and difficulty from each task's task.toml metadata +(the only task file read), with a fixed seed. Tasks named in --force-dev go +into the dev split regardless (tasks whose trajectories were already read +during development must never be held out). + +Usage: + python3 scripts/make-split.py [--dev 45] [--seed 20261015] [--force-dev fix-git ...] + +Refuses to overwrite an existing split: once committed it never changes. +""" + +import argparse +import json +import random +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +TASK_CACHE = Path.home() / ".cache" / "harbor" / "tasks" / "packages" / "terminal-bench" + + +def task_meta(task_dir: Path): + for version in task_dir.iterdir(): + toml = version / "task.toml" + if toml.is_file(): + text = toml.read_text() + section = "" + meta = {} + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + if line.startswith("["): + section = line.strip("[]") + elif section == "metadata" and "=" in line: + k, _, v = line.partition("=") + meta[k.strip()] = v.strip().strip('"') + return {"difficulty": meta.get("difficulty", "?"), "category": meta.get("category", "?")} + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--dev", type=int, default=45) + ap.add_argument("--seed", type=int, default=20261015) + ap.add_argument("--force-dev", nargs="*", default=[]) + ap.add_argument("--out", type=Path, default=REPO / "bench" / "split.json") + args = ap.parse_args() + if args.out.exists(): + sys.exit(f"{args.out} exists; the split never changes once committed") + + tasks = {} + for d in sorted(TASK_CACHE.iterdir()): + if d.is_dir() and re.match(r"^[a-z0-9.-]+$", d.name): + m = task_meta(d) + if m: + tasks[d.name] = m + if len(tasks) < 80: + sys.exit(f"only {len(tasks)} tasks in {TASK_CACHE}; download the dataset with harbor first") + unknown = [t for t in args.force_dev if t not in tasks] + if unknown: + sys.exit(f"unknown tasks: {unknown}") + + # Stratum = category + difficulty. Each stratum contributes a dev share + # proportional to its size; the remainder is settled by the seeded order. + rng = random.Random(args.seed) + strata = {} + for name, m in tasks.items(): + strata.setdefault((m["category"], m["difficulty"]), []).append(name) + dev = set(args.force_dev) + dev_share = args.dev / len(tasks) + leftovers = [] + for key in sorted(strata): + names = sorted(strata[key]) + rng.shuffle(names) + forced = [n for n in names if n in dev] + quota = dev_share * len(names) + want = int(quota) + leftovers.append((quota - want, key, [n for n in names if n not in dev])) + for n in names: + if len(forced) + (0 if n in dev else 0) >= want: + break + if n not in dev: + dev.add(n) + forced.append(n) + # Fill to exactly --dev from the strata with the largest fractional remainders. + leftovers.sort(key=lambda x: -x[0]) + for _, key, names in leftovers: + if len(dev) >= args.dev: + break + for n in names: + if n not in dev: + dev.add(n) + break + while len(dev) > args.dev: + # Over quota only if --force-dev exceeded it; drop a random non-forced task. + candidates = sorted(n for n in dev if n not in args.force_dev) + dev.remove(rng.choice(candidates)) + + heldout = sorted(set(tasks) - dev) + split = { + "dataset": "terminal-bench/terminal-bench-2-1", + "method": "stratified by task.toml category and difficulty; seed %d; forced dev: %s" % (args.seed, ", ".join(args.force_dev) or "none"), + "dev": sorted(dev), + "heldout": heldout, + } + args.out.write_text(json.dumps(split, indent=2) + "\n") + + def dist(names): + out = {} + for n in names: + out[tasks[n]["difficulty"]] = out.get(tasks[n]["difficulty"], 0) + 1 + return dict(sorted(out.items())) + + print(f"wrote {args.out}: {len(dev)} dev / {len(heldout)} held-out of {len(tasks)} tasks") + print("dev difficulty:", dist(dev), " held-out difficulty:", dist(heldout)) + cats = sorted({m['category'] for m in tasks.values()}) + for c in cats: + nd = sum(1 for n in dev if tasks[n]['category'] == c) + nh = sum(1 for n in heldout if tasks[n]['category'] == c) + print(f" {c:28s} dev {nd:2d} held-out {nh:2d}") + + +if __name__ == "__main__": + main() diff --git a/scripts/package-release.sh b/scripts/package-release.sh new file mode 100755 index 00000000..241603ea --- /dev/null +++ b/scripts/package-release.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Builds release archives into dist/: gitagent-.tar.gz for each +# target that can be built on this machine, plus SHA256SUMS. +# Linux targets need cargo-zigbuild when built on macOS; macOS targets need a Mac. +set -euo pipefail +cd "$(dirname "$0")/.." + +targets=("$@") +if [ ${#targets[@]} -eq 0 ]; then + targets=(x86_64-unknown-linux-musl aarch64-unknown-linux-musl) + if [ "$(uname -s)" = Darwin ]; then + targets+=(aarch64-apple-darwin x86_64-apple-darwin) + fi +fi + +rm -rf dist +mkdir -p dist +for target in "${targets[@]}"; do + rustup target add "$target" >/dev/null 2>&1 || true + case "$target" in + *-linux-*) scripts/build-release.sh "$target" ;; + *) cargo build --release --locked --target "$target" -p gitagent-cli ;; + esac + stage=$(mktemp -d) + cp "target/$target/release/gitagent" "$stage/gitagent" + cp README.md "$stage/" + tar -czf "dist/gitagent-$target.tar.gz" -C "$stage" gitagent README.md + rm -rf "$stage" + echo "packed dist/gitagent-$target.tar.gz" +done + +cd dist +if command -v sha256sum >/dev/null 2>&1; then + sha256sum gitagent-*.tar.gz > SHA256SUMS +else + shasum -a 256 gitagent-*.tar.gz > SHA256SUMS +fi +cat SHA256SUMS diff --git a/scripts/run-dev-pass.sh b/scripts/run-dev-pass.sh new file mode 100755 index 00000000..2e7912a2 --- /dev/null +++ b/scripts/run-dev-pass.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Run gitagent on the dev split, one Harbor job per agent time limit, all jobs at once. +# +# scripts/run-dev-pass.sh [profile.yaml] [skip-task ...] +# +# Jobs are named -t and written to bench/jobs. Only tasks +# listed under "dev" in bench/split.json are run; held-out tasks never are. +# gitagent's --timeout-s is the task's agent timeout minus 60 s, so it stops +# cleanly before Harbor does. Concurrency per job: N_900 (default 6) for the +# large 900 s group, N_OTHER (default 3) for the rest. K (default 1) runs each +# task K times. Keep N_900 + 6 * N_OTHER under about 27: each trial gets its own +# Docker network and the default address pools run out beyond that. +# To re-run trials that failed to start, export the key first (harbor job +# resume does not reload --env-file): +# set -a; . ./.env; set +a; harbor job resume -p bench/jobs/ -f +# +# SPLIT=heldout runs the held-out list instead. That is the final scoring run +# and happens once, on frozen code, so it also needs CONFIRM_HELDOUT=yes. +# SPLIT=all runs all 89 tasks (dev and held-out), the leaderboard format with +# K=5; it includes the held-out tasks, so it needs CONFIRM_HELDOUT=yes too. +set -euo pipefail + +prefix=${1:?usage: run-dev-pass.sh [profile.yaml] [skip-task ...]} +profile=${2:-bench/profiles/anthropic-m4.yaml} +shift $(( $# >= 2 ? 2 : 1 )) +skip=" $* " +model=${MODEL:-anthropic/claude-opus-5-5} +tasks_dir=${TASKS_DIR:-$HOME/.cache/harbor/tasks/packages/terminal-bench} + +cd "$(dirname "$0")/.." +export PATH="$HOME/.local/bin:$PATH" +[ -f .env ] || { echo "missing .env with ANTHROPIC_API_KEY" >&2; exit 1; } + +split=${SPLIT:-dev} +case $split in + dev) ;; + heldout|all) [ "${CONFIRM_HELDOUT:-}" = yes ] || { echo "$split includes the held-out tasks: set CONFIRM_HELDOUT=yes" >&2; exit 1; } ;; + *) echo "SPLIT must be dev, heldout or all" >&2; exit 1 ;; +esac +case $split in + all) selector='.dev[], .heldout[]' ;; + *) selector=".${split}[]" ;; +esac + +declare -A groups +for t in $(jq -r "$selector" bench/split.json); do + [[ $skip == *" $t "* ]] && continue + toml=$(ls "$tasks_dir/$t"/*/task.toml | head -1) + limit=$(awk '/^\[agent\]/{a=1;next} /^\[/{a=0} a && /^timeout_sec/{print int($3)}' "$toml") + groups[$limit]+=" -i terminal-bench/$t" +done + +pids=() +for limit in "${!groups[@]}"; do + n=${N_OTHER:-3}; [ "$limit" = 900 ] && n=${N_900:-6} + job="$prefix-t$limit" + echo "$job: $(wc -w <<<"${groups[$limit]}" | awk '{print $1/2}') task(s), timeout_s=$((limit - 60)), -n $n" + PYTHONPATH=$PWD harbor run -d terminal-bench/terminal-bench-2-1 -a gitagent_harbor.agent:GitAgent \ + -m "$model" --env-file .env --ak timeout_s=$((limit - 60)) --ak profile="$PWD/$profile" \ + -k "${K:-1}" -n "$n" -o bench/jobs --job-name "$job" -q ${groups[$limit]} > "bench/jobs/.$job.log" 2>&1 & + pids+=($!) +done +status=0 +for p in "${pids[@]}"; do wait "$p" || status=1; done +exit $status From df0fea60230560a0aad0370d76b4d9f5b3c1de1a Mon Sep 17 00:00:00 2001 From: chandan25809-lyzr Date: Tue, 20 Oct 2026 22:13:15 +0000 Subject: [PATCH 02/82] bench: register the Terminal-Bench 2.1 full run (89 x 5, Opus 5.5, no lessons) --- bench/rounds.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bench/rounds.json b/bench/rounds.json index 80cc9e83..4e88e2b4 100644 --- a/bench/rounds.json +++ b/bench/rounds.json @@ -214,6 +214,24 @@ "Learning is off: the learner refuses held-out tasks; the result is not used to change code" ], "reasoning": "37/44 (84.1%; 86.0% without qemu-startup, which this machine cannot run; est. cost $51.72). Two of the seven failures were not attempts: on break-filter-js-from-html and vulnerable-secret, OpenAI's API stopped the run with a cyber_policy refusal (content flagged for possible cybersecurity risk; OpenAI offers a separate access programme for authorised security work). Without those two the model solved 37 of 42 (88.1%); without them and qemu-startup, 37 of 41 (90.2%). The other failures: qemu-startup (machine; 0/3 for Opus too), video-processing (2/3 for Opus), and three that Opus solved in all three of its held-out runs: extract-elf, sanitize-git-repo, torch-pipeline-parallelism. One run per task, so about +/-3 points of noise; lower than GPT-6's dev rounds (88.9%), with the refusals accounting for most of the gap. Not comparable to the official leaderboard (89 tasks x 5 runs), and not used to change code." + }, + { + "name": "tb21-final", + "split": "all", + "model": "claude-opus-5-5", + "label": "Terminal-Bench 2.1 full run for publication: all 89 tasks x 5 runs, Claude Opus 5.5, effort high, no lessons", + "host": "optimus", + "commit": "79e4d28", + "lessons_commit": null, + "jobs": [ + "tb21-final-opus55-*" + ], + "changes": [ + "Leaderboard format: every task, 5 trials each, the tasks' own time limits and resources", + "Profile bench/profiles/anthropic-m4-nolessons.yaml: the M4 profile with learned lessons switched off, so the agent sees only the system prompt", + "Code frozen at the published commit 79e4d28; results uploaded to Harbor Hub" + ], + "reasoning": "" } ] } From e52ad3b9930f941bdb9e8467270ecd5c80f4e4fb Mon Sep 17 00:00:00 2001 From: chandan25809-lyzr Date: Wed, 21 Oct 2026 16:18:21 +0000 Subject: [PATCH 03/82] Dashboard: Harbor's job records (mean, pass@k, trial counts, config, timeout multipliers, task versions, Hub link) and a compliance strip per round; Harbor's per-trial log is no longer shown --- bench/hub.json | 1 + scripts/dashboard.py | 352 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 334 insertions(+), 19 deletions(-) create mode 100644 bench/hub.json diff --git a/bench/hub.json b/bench/hub.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/bench/hub.json @@ -0,0 +1 @@ +{} diff --git a/scripts/dashboard.py b/scripts/dashboard.py index bd673dce..427f4523 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -14,7 +14,7 @@ Usage: python3 scripts/dashboard.py [--root bench/jobs] [--port 8731] -Stdlib only (Python 3.9+); binds to 127.0.0.1. +Stdlib only (Python 3.9+); binds to localhost only. """ import argparse @@ -425,6 +425,43 @@ def newest_mtime(trial_dir: Path): return newest +TIMEOUT_MULTIPLIERS = ("timeout_multiplier", "agent_timeout_multiplier", "verifier_timeout_multiplier", + "agent_setup_timeout_multiplier", "environment_build_timeout_multiplier") +TIMEOUT_OVERRIDES = (("agent", "override_timeout_sec"), ("agent", "max_timeout_sec"), ("agent", "override_setup_timeout_sec"), + ("verifier", "override_timeout_sec"), ("verifier", "max_timeout_sec")) + + +def effective_multipliers(raw): + """Each timeout multiplier as Harbor applies it: an unset per-phase one + falls back to timeout_multiplier (itself 1.0 when unset).""" + if not raw: + return None + base = raw.get("timeout_multiplier") + base = 1.0 if base is None else base + return {k: (base if raw.get(k) is None else raw[k]) for k in TIMEOUT_MULTIPLIERS} + + +def trial_harbor(result, cfg): + """What Harbor recorded about how a trial was run: the task's version and + checksum, the verifier mode and the timeout settings. The settings come + from result.json's config, or the trial's config.json while it runs.""" + result = result if isinstance(result, dict) else {} + tcfg = result.get("config") if isinstance(result.get("config"), dict) else (cfg if isinstance(cfg, dict) else {}) + task_id = result.get("task_id") if isinstance(result.get("task_id"), dict) else {} + raw = {k: tcfg.get(k) for k in TIMEOUT_MULTIPLIERS} if tcfg else None + overrides = {f"{sec}.{key}": (tcfg.get(sec) or {}).get(key) for sec, key in TIMEOUT_OVERRIDES + if isinstance(tcfg.get(sec), dict) and (tcfg.get(sec) or {}).get(key) is not None} + return { + "task_checksum": result.get("task_checksum"), + "task_id": {k: task_id.get(k) for k in ("org", "name", "ref")} if task_id else None, + "source": result.get("source"), + "verifier_environment_mode": result.get("verifier_environment_mode"), + "timeout_multipliers": raw, + "effective_multipliers": effective_multipliers(raw), + "timeout_overrides": overrides, + } + + def read_trial(trial_dir: Path, with_live=True, job_running=False): """One row per trial, from result.json and gitagent's own agent/ files.""" result = load_json(trial_dir / "result.json") @@ -469,6 +506,9 @@ def read_trial(trial_dir: Path, with_live=True, job_running=False): "live": None, "counters": None, "cost_usd": None, + # Harbor's per-trial record (task version, timeout settings); scan() and + # trial_detail() take it out of the row before it is sent. + "_harbor": trial_harbor(result, cfg), } if result is None: @@ -566,10 +606,168 @@ def round_of(job_name, rounds): return None +# bench/hub.json (optional): {"": ""}; `harbor upload` leaves nothing in the job dir once it succeeds, so the link is recorded there by hand. +HUB_FILE = REPO / "bench" / "hub.json" +# The .harbor-upload/ folder holds Harbor's upload archive only while an upload runs; one left behind means it did not finish. +UPLOAD_SCRATCH = ".harbor-upload" +# The difference, in points, between Harbor's mean reward and the dashboard's accuracy that gets a warning. +MEAN_TOLERANCE_PTS = 0.5 + + +def hub_links(): + data = load_json(HUB_FILE) + if not isinstance(data, dict): + return {} + return {k: v for k, v in data.items() if isinstance(v, str) and re.match(r"^https?://", v)} + + +def harbor_job(job_dir: Path, trials, metas, hub): + """Harbor's own record of a job, from its result.json and config.json: + counts, mean reward and pass@k per agent/model/dataset, and the settings + the job ran with (Harbor leaves out settings at their default value).""" + res = load_json(job_dir / "result.json") + cfg = load_json(job_dir / "config.json") + # Harbor writes the job's result.json while it runs; finished_at is set at the end. + finished = isinstance(res, dict) and bool(res.get("finished_at")) + res = res if finished else {} + cfg = cfg if isinstance(cfg, dict) else {} + stats = res.get("stats") or {} + + agents_cfg = cfg.get("agents") + agents = [] + for a in agents_cfg if isinstance(agents_cfg, list) else [{}]: + a = a if isinstance(a, dict) else {} + kwargs = a.get("kwargs") if isinstance(a.get("kwargs"), dict) else {} + agents.append({ + "name": a.get("import_path") or a.get("name") or "oracle", + "model_name": a.get("model_name"), + "model_id": model_id(a.get("model_name")), + "kwargs": kwargs, + "profile": kwargs.get("profile"), + "timeout_s": kwargs.get("timeout_s"), + }) + datasets = [] + for ds in cfg.get("datasets") or []: + if isinstance(ds, dict): + names = ds.get("task_names") + datasets.append({"name": ds.get("name") or ds.get("path"), "ref": ds.get("ref"), + "n_tasks": len(names) if isinstance(names, list) and names else None}) + n_attempts = cfg.get("n_attempts") + attempts = n_attempts if isinstance(n_attempts, int) else 1 + listed = [d["n_tasks"] for d in datasets] + tasks_known = bool(listed) and all(n is not None for n in listed) + n_tasks = sum(listed) if tasks_known else len({t["task"] for t in trials}) + + evals = [] + for key, e in (stats.get("evals") or {}).items(): + e = e if isinstance(e, dict) else {} + parts = key.split("__") + agent, dataset = parts[0], parts[-1] + model = parts[1] if len(parts) >= 3 else None + mine = [t for t in trials if t["kind"] == agent_kind(agent) and (t["model_id"] or None) == (model or None)] + if not mine and len(stats.get("evals") or {}) == 1: + mine = trials + scored = [t for t in mine if t["reward"] is not None] + passed = sum(1 for t in scored if t["reward"] >= 1.0) + mean = next((m.get("mean") for m in e.get("metrics") or [] if isinstance(m, dict) and m.get("mean") is not None), None) + dash = passed / len(scored) if scored else None + diff = round(100 * (mean - dash), 2) if mean is not None and dash is not None else None + evals.append({ + "key": key, "agent": agent, "model": model, "dataset": dataset, + "n_trials": e.get("n_trials"), "n_errors": e.get("n_errors"), "mean": mean, + "pass_at_k": {str(k): v for k, v in sorted((e.get("pass_at_k") or {}).items(), key=lambda kv: int(kv[0]) if str(kv[0]).isdigit() else 0)}, + "exceptions": {k: len(v) if isinstance(v, list) else v for k, v in (e.get("exception_stats") or {}).items()}, + "dash_passed": passed, "dash_scored": len(scored), "dash_accuracy": dash, + "diff_pts": diff, "differs": diff is not None and abs(diff) > MEAN_TOLERANCE_PTS, + }) + + multipliers, overrides, versions = {}, {}, {} + for t, m in zip(trials, metas): + for k, v in (m.get("effective_multipliers") or {}).items(): + multipliers.setdefault(k, set()).add(v) + for k, v in (m.get("timeout_overrides") or {}).items(): + overrides.setdefault(k, set()).add(v) + versions.setdefault(t["task"], set()) + if m.get("task_checksum"): + versions[t["task"]].add(m["task_checksum"]) + count = lambda k: stats.get(k) if finished else None # noqa: E731 + return { + "id": res.get("id"), + "finished": finished, + "started_at": res.get("started_at"), + "finished_at": res.get("finished_at"), + "n_total_trials": res.get("n_total_trials"), + "completed": count("n_completed_trials"), + "errored": count("n_errored_trials"), + "running": count("n_running_trials"), + "pending": count("n_pending_trials"), + "cancelled": count("n_cancelled_trials"), + "retries": count("n_retries"), + "n_attempts": attempts, + "n_attempts_default": not isinstance(n_attempts, int), + "n_concurrent_trials": cfg.get("n_concurrent_trials") if isinstance(cfg.get("n_concurrent_trials"), int) else 4, + "n_concurrent_default": not isinstance(cfg.get("n_concurrent_trials"), int), + "agents": agents, + "agents_default": not isinstance(agents_cfg, list), + "datasets": datasets, + "n_tasks": n_tasks, + "tasks_listed": tasks_known, + "expected_trials": n_tasks * attempts * max(len(agents), 1), + "present_trials": len(trials), + "evals": evals, + "multipliers": {k: sorted(v) for k, v in multipliers.items()}, + "timeout_overrides": {k: sorted(v) for k, v in overrides.items()}, + "task_versions": max((len(v) for v in versions.values()), default=0), + "hub_url": hub.get(job_dir.name), + "upload_unfinished": (job_dir / UPLOAD_SCRATCH).is_dir(), + # Internal: the checksums seen per task, for the round's version check; summary() drops it. + "_versions": {k: sorted(v) for k, v in versions.items()}, + } + + +def round_compliance(jobs): + """The round's run settings against what a publishable result needs: the + same trials per task everywhere, timeouts unscaled, nothing retried or + errored, every expected trial present, and one version of each task.""" + if not jobs: + return None + hs = [j["harbor"] for j in jobs] + mult, over, versions = {}, {}, {} + for h in hs: + for k, v in h["multipliers"].items(): + mult.setdefault(k, set()).update(v) + for k, v in h["timeout_overrides"].items(): + over.setdefault(k, set()).update(v) + for task, sums in h["_versions"].items(): + versions.setdefault(task, set()).update(sums) + num = lambda k: sum(h[k] or 0 for h in hs) # noqa: E731 + return { + "jobs": len(hs), + "unfinished_jobs": sorted(j["name"] for j in jobs if not j["harbor"]["finished"]), + "n_attempts": sorted({h["n_attempts"] for h in hs}), + "n_attempts_defaulted": sorted(j["name"] for j in jobs if j["harbor"]["n_attempts_default"]), + "multipliers": {k: sorted(v) for k, v in mult.items()}, + "multipliers_ok": bool(mult) and all(x == 1.0 for v in mult.values() for x in v) and not over, + "timeout_overrides": {k: sorted(v) for k, v in over.items()}, + "retries": num("retries"), + "errored": num("errored"), + "cancelled": num("cancelled"), + "present": num("present_trials"), + "expected": num("expected_trials"), + "expected_exact": all(h["tasks_listed"] for h in hs), + "harbor_total": num("n_total_trials"), + "tasks": len(versions), + "multi_version_tasks": sorted(t for t, v in versions.items() if len(v) > 1), + "unversioned_tasks": sorted(t for t, v in versions.items() if not v), + "max_versions": max((len(v) for v in versions.values()), default=0), + } + + def scan(root: Path): jobs = [] rounds = load_rounds() round_model = {r["name"]: r.get("model") for r in rounds} + hub = hub_links() try: job_list = sorted((d for d in root.iterdir() if d.is_dir()), key=lambda d: d.stat().st_mtime, reverse=True) except OSError: @@ -580,6 +778,7 @@ def scan(root: Path): trials = [read_trial(d, job_running=job_running) for d in trial_dirs(job_dir)] if not trials: continue + metas = [t.pop("_harbor") for t in trials] job_round = round_of(job_dir.name, rounds) declared = round_model.get(job_round) for t in trials: @@ -610,6 +809,7 @@ def scan(root: Path): "errors": sum(1 for t in trials if t["status"] == "error"), "cost_usd": round(sum(t["cost_usd"] or 0 for t in trials), 2), "unpriced_runs": sum(1 for t in trials if t["cost_usd"] is None and (t["input_tokens"] or 0) > 0), + "harbor": harbor_job(job_dir, trials, metas, hub), "trials": trials, } ) @@ -924,9 +1124,10 @@ def lessons_at(commit): return _lessons_cache[commit] -def rounds_page(progress): +def rounds_page(progress, jobs=()): """Each round with what it did, why it scored as it did, its accuracy per - harness, and the lessons its binary carried compared with the previous round's.""" + harness, the lessons its binary carried compared with the previous round's, + and how its Harbor jobs were run (the compliance strip).""" out, prev = [], None for r in load_rounds(): if not r.get("counts", True): @@ -938,7 +1139,8 @@ def rounds_page(progress): diff = {"added": [l for l in lessons if l not in prev], "dropped": [l for l in prev if l not in lessons]} out.append({k: r.get(k) for k in ("name", "label", "host", "commit", "changes", "reasoning", "lessons_commit", "model")} | {"split": "held-out" if r.get("split") == "heldout" else "dev", - "results": rows, "lessons": lessons, "lessons_diff": diff}) + "results": rows, "lessons": lessons, "lessons_diff": diff, + "compliance": round_compliance([j for j in jobs if j["round"] == r["name"]])}) if lessons is not None: prev = lessons return out @@ -1074,11 +1276,14 @@ def summary(root: Path): compare = dict(by_model_cmp[primary]) if primary else dict(comparison([], task_meta), gitagent_round=None) compare.update(models=model_ids, default_model=primary, by_model=by_model_cmp) lessons = load_json(REPO / "bench" / "lessons" / "lessons.json") + rounds_view = rounds_page(progress, jobs) + for job in jobs: + job["harbor"].pop("_versions", None) return { "models": {m: model_info(m, i, has_rounds) for i, (m, has_rounds) in enumerate(models)}, "compare": compare, "progress": progress, - "rounds_page": rounds_page(progress), + "rounds_page": rounds_view, "leaderboard": load_json(REPO / "bench" / "leaderboard.json") or {}, "generated_at": datetime.now(timezone.utc).isoformat(), "root": str(root), @@ -1147,6 +1352,7 @@ def trial_detail(root: Path, job: str, trial: str): return None result = load_json(trial_dir / "result.json") or {} row = read_trial(trial_dir) + task_record = row.pop("_harbor", None) or {} detail = {"row": row, "timings": [], "steps": [], "events": [], "turns": [], "files": [], "log_tail": ""} for key in ("environment_setup", "agent_setup", "agent_execution", "verifier"): @@ -1225,11 +1431,12 @@ def trial_detail(root: Path, job: str, trial: str): exc = result.get("exception_info") detail["harbor"] = { "exception": exc if isinstance(exc, dict) else ({"exception_message": str(exc)} if exc else None), - "trial_log": read_text(trial_dir / "trial.log", 64 * 1024), "done": (read_text(agent_dir / "done", 64) or "").strip(), "has_trajectory": (agent_dir / "trajectory.json").is_file(), "has_log": (agent_dir / "gitagent.log").is_file(), } + # Task version and run settings from the trial's result.json (see trial_harbor). + detail["harbor"].update(task_record) return detail @@ -1307,6 +1514,7 @@ def trial_file(root: Path, job: str, trial: str, rel: str): .chip b { color:var(--ink); font-weight:600; font-variant-numeric:tabular-nums; } .chip.chip-alarm { border-color:var(--fail); background:var(--fail-bg); color:var(--fail); } .chip.chip-alarm b { color:var(--fail); } +.chips.strip { margin:0 0 var(--s3); } /* ---- tabs ---- */ .tabs { position:sticky; top:0; z-index:20; display:flex; flex-wrap:wrap; gap:2px; background:var(--bg); @@ -1486,9 +1694,9 @@ def trial_file(root: Path, job: str, trial: str, rel: str):

Accuracy

Per model; a card never averages two models together.

Tokens and cost (all model-calling runs)

Money adds up across models; tokens, cache shares and per-run costs are shown per model because prices and caching differ between providers.

Running now

-

Jobs

One job is one harbor run. Oracle jobs run the tasks' reference solutions to check the machine, not the agent; terminus jobs are the baseline agent. Click a job to see its trials; click a heading to sort.

+

Jobs

One job is one harbor run. Oracle jobs run the tasks' reference solutions to check the machine, not the agent; terminus jobs are the baseline agent. "Harbor mean" is Harbor's own mean reward from the job's result.json; a warn badge marks a gap of more than 0.5 points from the accuracy beside it. Click a job to see its trials and Harbor's record of it; click a heading to sort.

-
JobStartedAgentModelProfileTrialsPassAccuracyEst. cost
+ TrialsPassAccuracyHarbor meanEst. cost

Accuracy by task, per harness and model

One row per task, harness and model (oracle runs are the machine check, shown alongside). Rows of different models are never averaged.

TaskDifficultyHarnessModelRunsPassedAccuracyOracle
@@ -1517,6 +1725,7 @@ def trial_file(root: Path, job: str, trial: str, rel: str):