From a22cb4d807e517289168c619fc0b76d6fc05ae24 Mon Sep 17 00:00:00 2001 From: Chang Luo <33987852+luochang212@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:12:48 +0800 Subject: [PATCH 1/2] feat(autoresearch): update to 0.2.0 --- external_plugins/autoresearch/.mcp.json | 7 +- .../autoresearch/.zcode-plugin/plugin.json | 3 +- external_plugins/autoresearch/README.md | 96 +- external_plugins/autoresearch/README_CN.md | 96 +- .../autoresearch/commands/autoresearch.md | 8 +- .../autoresearch/commands/clear.md | 6 +- .../autoresearch/commands/export.md | 4 +- .../autoresearch/commands/finalize.md | 5 +- external_plugins/autoresearch/commands/off.md | 6 +- .../hooks/examples/after/macos-notify.sh | 7 +- .../hooks/examples/before/anti-thrash.sh | 7 +- .../hooks/examples/before/idea-rotator.sh | 51 +- .../hooks/examples/before/self-check.sh | 38 + .../autoresearch/hooks/guard-frozen.ts | 102 +- .../autoresearch/hooks/hooks.json | 22 +- .../autoresearch/hooks/memory-inject.ts | 55 +- .../autoresearch/hooks/permission-gate.ts | 54 - .../autoresearch/hooks/session-start.ts | 12 +- .../autoresearch/hooks/stop-continue.ts | 143 +- external_plugins/autoresearch/mcp/lib/bash.ts | 180 ++ .../autoresearch/mcp/lib/dashboard-server.ts | 83 +- .../autoresearch/mcp/lib/dashboard.ts | 656 +++++-- .../autoresearch/mcp/lib/experiment.ts | 117 +- .../autoresearch/mcp/lib/frozen.ts | 40 + external_plugins/autoresearch/mcp/lib/git.ts | 342 +++- .../autoresearch/mcp/lib/harness.ts | 47 + .../autoresearch/mcp/lib/ledger.ts | 218 ++- .../autoresearch/mcp/lib/paths.ts | 50 +- .../autoresearch/mcp/lib/spill.ts | 42 + .../autoresearch/mcp/lib/types.ts | 75 +- .../autoresearch/mcp/lib/validate.ts | 27 +- external_plugins/autoresearch/mcp/server.ts | 1121 +++++++++-- .../autoresearch/scripts/finalize.sh | 28 +- .../autoresearch/scripts/launcher.mjs | 230 +++ .../skills/autoresearch-hooks/SKILL.md | 9 +- .../autoresearch/skills/autoresearch/SKILL.md | 43 +- .../autoresearch/references/loop-protocol.md | 97 +- .../autoresearch/references/setup-guide.md | 56 +- .../autoresearch/references/unattended.md | 8 +- .../autoresearch/tests/bash.test.ts | 124 ++ .../tests/dashboard-server.test.ts | 117 ++ .../autoresearch/tests/dashboard.test.ts | 426 ++++- .../autoresearch/tests/examples.test.ts | 142 +- .../autoresearch/tests/experiment.test.ts | 193 +- .../autoresearch/tests/finalize.test.ts | 55 +- .../autoresearch/tests/frozen.test.ts | 40 + .../autoresearch/tests/git.test.ts | 420 +++++ .../autoresearch/tests/harness-trace.test.ts | 400 ++++ .../autoresearch/tests/hooks.test.ts | 446 ++++- .../autoresearch/tests/launcher.test.ts | 295 +++ .../autoresearch/tests/ledger.test.ts | 217 ++- .../tests/mcp-integration.test.ts | 1666 ++++++++++++++++- .../autoresearch/tests/paths.test.ts | 72 + .../tests/platform-guards.test.ts | 162 ++ .../autoresearch/tests/spill.test.ts | 39 + .../autoresearch/tests/validate.test.ts | 18 + .../autoresearch/tests/win32-kill.test.ts | 234 +++ marketplace.json | 2 +- 58 files changed, 8304 insertions(+), 955 deletions(-) create mode 100755 external_plugins/autoresearch/hooks/examples/before/self-check.sh delete mode 100644 external_plugins/autoresearch/hooks/permission-gate.ts create mode 100644 external_plugins/autoresearch/mcp/lib/bash.ts create mode 100644 external_plugins/autoresearch/mcp/lib/frozen.ts create mode 100644 external_plugins/autoresearch/mcp/lib/harness.ts create mode 100644 external_plugins/autoresearch/mcp/lib/spill.ts create mode 100644 external_plugins/autoresearch/scripts/launcher.mjs create mode 100644 external_plugins/autoresearch/tests/bash.test.ts create mode 100644 external_plugins/autoresearch/tests/dashboard-server.test.ts create mode 100644 external_plugins/autoresearch/tests/frozen.test.ts create mode 100644 external_plugins/autoresearch/tests/harness-trace.test.ts create mode 100644 external_plugins/autoresearch/tests/launcher.test.ts create mode 100644 external_plugins/autoresearch/tests/paths.test.ts create mode 100644 external_plugins/autoresearch/tests/platform-guards.test.ts create mode 100644 external_plugins/autoresearch/tests/spill.test.ts create mode 100644 external_plugins/autoresearch/tests/win32-kill.test.ts diff --git a/external_plugins/autoresearch/.mcp.json b/external_plugins/autoresearch/.mcp.json index 719b7e98..409ea18e 100644 --- a/external_plugins/autoresearch/.mcp.json +++ b/external_plugins/autoresearch/.mcp.json @@ -3,7 +3,10 @@ "autoresearch": { "type": "stdio", "command": "node", - "args": ["${ZCODE_PLUGIN_ROOT}/mcp/server.ts"], + "args": [ + "${ZCODE_PLUGIN_ROOT}/scripts/launcher.mjs", + "${ZCODE_PLUGIN_ROOT}/mcp/server.ts" + ], "cwd": "${ZCODE_PROJECT_DIR}", "env": { "AR_MAX_ITERATIONS": "${user_config.maxIterations}", @@ -11,7 +14,7 @@ "AR_CHECKS_TIMEOUT_MS": "${user_config.checksTimeoutMs}" }, "enabled": true, - "timeoutMs": 60000 + "timeoutMs": 600000 } } } diff --git a/external_plugins/autoresearch/.zcode-plugin/plugin.json b/external_plugins/autoresearch/.zcode-plugin/plugin.json index 9fd286eb..c45abc1d 100644 --- a/external_plugins/autoresearch/.zcode-plugin/plugin.json +++ b/external_plugins/autoresearch/.zcode-plugin/plugin.json @@ -5,10 +5,11 @@ "en": "Autonomous experiment loop for ZCode: set a goal, pick a mechanical metric, and let the agent iterate. It measures, keeps what works, reverts what doesn't, and repeats.", "zh-CN": "ZCode 自主实验循环：设定目标与机械度量，让 agent 迭代。每一轮测量、保留有效改动、回滚无效改动，如此往复。" }, - "version": "0.1.0", + "version": "0.2.0", "author": { "name": "luochang212" }, + "skills": "./skills", "keywords": [ "autoresearch", "experiment", diff --git a/external_plugins/autoresearch/README.md b/external_plugins/autoresearch/README.md index 3390ba21..f908ff84 100644 --- a/external_plugins/autoresearch/README.md +++ b/external_plugins/autoresearch/README.md @@ -4,69 +4,73 @@ Let the ZCode coding agent iterate autonomously on a fixed, mechanical metric: modify code → run the benchmark → keep improvements, revert regressions → repeat. -Based on research into [karpathy/autoresearch](https://github.com/karpathy/autoresearch) and [pi-autoresearch](https://github.com/davebcn87/pi-autoresearch) (see `docs/research/autoresearch-survey.md`). Architecture decisions live in `adr/decisions/`. +Based on research into [karpathy/autoresearch](https://github.com/karpathy/autoresearch) and [pi-autoresearch](https://github.com/davebcn87/pi-autoresearch) (see `docs/research/autoresearch-survey.md` at the repository root). Architecture decisions live in `adr/implemented/`. ## Security and side effects This plugin executes code and operates on a git repository. Enabling it grants code-execution trust (official marketplace convention). Specifically, it: - **Runs commands**: `run_experiment` executes the benchmark script you author (`.auto/measure.sh`) and, when present, the correctness gate (`.auto/checks.sh`); -- **Runs git operations automatically**: `git commit` on keep, automatic rollback on non-keep (`.auto/` is exempt from rollback); -- **Installs ZCode hooks**: Stop (loop continuation), PreToolUse (frozen-file write protection), PermissionRequest (experiment-tool gating), UserPromptSubmit/SessionStart (ledger memory injection); -- **Serves a local HTTP dashboard** on 127.0.0.1 via `export_dashboard`; +- **Runs git operations automatically**: `git commit` on keep, automatic rollback on discard/crash/checks_failed (noop changes nothing; session state: `.auto/`, the host session dir `.zcode/`, and `autoresearch-dashboard.html`, is exempt from commit/rollback); +- **Installs ZCode hooks**: Stop (loop continuation), PreToolUse (frozen-file write protection), UserPromptSubmit (ledger memory injection), SessionStart (resume hint); +- **Serves a local HTTP dashboard** on 127.0.0.1 via `export_dashboard` (editorial archive page: full experiment history, trend across segments, light/dark themes); - **Writes session state** to `.auto/` files (`log.jsonl`, `config.json`) in the project directory. No third-party npm dependencies: the MCP server and hooks are Node-stdlib TypeScript scripts (Node ≥24, types stripped natively, no build step). +Node ≥24 must exist on the machine, but it does not have to be on the host's PATH: the MCP server and hooks launch through `scripts/launcher.mjs` via plain `node` (the one command the host resolves on all platforms), which resolves Node from PATH, then nvm (POSIX `$NVM_DIR` / Windows `%NVM_HOME%` or `%APPDATA% +vm`, newest-first), Homebrew or `/usr/local/bin` (POSIX) / `%ProgramFiles% +odejs` (Windows), and honors `AUTORESEARCH_NODE_BIN` to pin one explicitly (run `node scripts/launcher.mjs --resolve` to see what it picks). The benchmark scripts (`.auto/measure.sh`, `.auto/checks.sh`) are still bash and need a POSIX shell (Git-Bash or WSL) on Windows. + ## Install This repository is itself a plugin marketplace (`marketplace.json` points to `./plugin`). In ZCode: 1. Add the marketplace: this repository's URL (or a local directory). 2. In **Settings → Plugin Management**, install and enable `autoresearch`. -3. The plugin provides: an MCP server (5 tools), the `autoresearch` skill, 5 slash commands, and 5 hooks. +3. The plugin provides: an MCP server (5 tools), 2 skills (`autoresearch` + `autoresearch-hooks`), 5 slash commands, and 4 hooks. ## Usage ``` /autoresearch:autoresearch # enter/resume autoresearch mode (runs setup if there is no session) -/autoresearch:export # export a static dashboard (autoresearch-dashboard.html) -/autoresearch:off # pause loop continuation (sets autoresearchOff: true) +/autoresearch:export # export the dashboard (live SSE URL + static autoresearch-dashboard.html) +/autoresearch:off # pause the loop and its hints (sets autoresearchOff: true) /autoresearch:clear # reset the session ledger /autoresearch:finalize # organize kept experiments into a clean branch (scripts/finalize.sh) ``` Or let the skill trigger on its own (descriptions containing "autoresearch", "autonomous optimization", etc.). A full loop: -1. **Setup**: pick a mechanical metric → write `.auto/measure.sh` (emits `METRIC name=value` lines) → optionally `.auto/checks.sh` (correctness gate) → write the `.auto/prompt.md` charter → create an experiment branch `git checkout -b autoresearch/`. +1. **Setup**: pick a mechanical metric → write `.auto/measure.sh` (emits `METRIC name=value` lines) → optionally `.auto/checks.sh` (correctness gate) → record the user's deep goal verbatim in `.auto/goal.md` and the agreed scope boundary in `.auto/scope.md` (both frozen; the hook denies rewrites) → write the `.auto/prompt.md` charter (its Goal/Scope sections quote the two files verbatim) → create an experiment branch `git checkout -b autoresearch/`. 2. `init_experiment` (metric_name, direction) → run a baseline. -3. **Loop**: one focused change → `run_experiment` → `log_experiment` (keep auto-commits / non-keep auto-rolls-back, `.auto/` exempt). +3. **Loop**: one focused change → `run_experiment` → `log_experiment` (keep auto-commits; discard/crash/checks_failed auto-roll-back; noop changes nothing; `.auto/`, the host session dir `.zcode/`, and the dashboard are exempt from both). ## Tools (MCP) -| Tool | What it does | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `init_experiment` | Start/restart an experiment segment (name, primary metric, direction lower/higher) | -| `run_experiment` | Run the benchmark: times it, parses `METRIC name=value` lines, returns a truncated tail (10 lines / 4KB), kills the process group on timeout, takes the median over `repeat` runs, runs the `before.sh` hook | -| `log_experiment` | Record the outcome: keep auto-commits (`experiment:` prefix); non-keep auto-rolls-back (`.auto/` exempt); returns baseline/best/delta/confidence/plateau plus a next-action hint, with a `revisit_nudge` when the segment has a rollback-reasoned discard worth re-checking (annotate deliberate retries with `asi.revisits_run: `); the computed confidence is snapshotted into the ledger run row; runs the `after.sh` hook | -| `export_dashboard` | Serve a live local dashboard (127.0.0.1 + SSE auto-refresh) and write a static HTML fallback | -| `clear_experiments` | Delete `.auto/log.jsonl` and reset the session (keeps measure/checks/prompt) | +| Tool | What it does | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init_experiment` | Start/restart an experiment segment (name, primary metric, direction lower/higher); optional loop calibration: `plateau_min_improvement` / `plateau_window` (set near the metric's real step size to avoid false plateaus) and `deterministic` (hints skip repeat re-measurement) | +| `run_experiment` | Run the benchmark: `command` is optional when `.auto/measure.sh` exists (defaults to it), required in free-form mode; times it, parses `METRIC name=value` lines, returns a truncated tail (10 lines / 4KB), kills the process group on timeout, takes the median over `repeat` runs, runs the `before.sh` hook | +| `log_experiment` | Record the outcome: keep auto-commits (`experiment:` prefix); discard/crash/checks_failed auto-roll back (noop: nothing to roll back; session state `.auto/` + `.zcode/` + dashboard exempt); reports a discard/crash rollback via `rolled_back` (restored_to hash + file list), returns baseline/best/delta/confidence (unit × noise)/plateau/`runs_left` plus a next-action hint, with a `revisit_nudge` when a discard/checks_failed lands and the segment has a rollback-reasoned discard worth re-checking (annotate deliberate retries with `asi.revisits_run: `, and environment-caused crashes with `asi.infra: true` — infra rows are excluded from the doom-loop signal); the computed confidence is snapshotted into the ledger run row; runs the `after.sh` hook | +| `export_dashboard` | Serve a live local dashboard (127.0.0.1 + SSE auto-refresh) and write a static HTML fallback | +| `clear_experiments` | Delete `.auto/log.jsonl` and reset the session (all other `.auto/` files are kept) | ## Guardrails -- **Benchmark locking**: when `.auto/measure.sh` exists, `run_experiment` only executes that script (validated after stripping env/time/nice wrappers). -- **Correctness backpressure**: when `.auto/checks.sh` exists, it runs automatically after a passing benchmark; a failing gate forbids keep and rolls back. -- **Write protection**: a PreToolUse hook denies writes to `.auto/measure.sh` / `.auto/checks.sh`. -- **Tool gating (approximate)**: a PermissionRequest hook denies experiment tools when there is no session (only covers calls that go through the permission prompt). -- **Auto-resume hint**: SessionStart injects a continuation hint when an active session is detected; `/autoresearch:off` pauses it (`autoresearchOff: true`). -- **Memory injection**: UserPromptSubmit/SessionStart hooks inject an aggregated summary (progress + deduped tried directions + best trajectory + discard reasons with rollback rationale + ASI distillation) so progress survives compaction; repeated/oscillating attempts (doom-loop) trigger a hint to switch direction. -- **Loop continuation**: the Stop hook blocks (`decision:block`) while a loop is unfinished (zcode platform limit: 3 consecutive windows). +- **Benchmark locking**: when `.auto/measure.sh` exists, `run_experiment` only executes that script (validated after stripping env/time/nice wrappers and leading assignments; assignment values pass the same character whitelist, so `$(`/backtick smuggling is rejected). +- **Correctness backpressure**: when `.auto/checks.sh` exists, it runs automatically after a passing benchmark; a failing gate forbids keep and rolls back. A gate that never produces a verdict (not executable, killed by a signal, exits ≥126) is infrastructure, not failure — it does not veto keeps and is surfaced as `verdict: false`. +- **Write protection**: a PreToolUse hook denies Write/Edit/ApplyPatch rewrites (ApplyPatch's per-file targets are parsed from the patch text) of `.auto/measure.sh` / `.auto/checks.sh` / `.auto/goal.md` (the deep-goal record: the root the loop serves stays immutable while the harness-edit step revises its surface representations) / `.auto/scope.md` (the user-agreed scope boundary: the loop works inside the fence, it never redraws it — widening or narrowing is the user's edit or a new segment). First creation during setup is the intended exception — frozen means immutable once recorded. A denied rewrite attempt is recorded to the ledger as a `frozen_attempt` row. +- **Harness trace**: the editable surfaces (`prompt.md`, `ideas.md`, `experience.md`, `.auto/hooks/before.sh` / `after.sh`) are hashed at the accounting checkpoints (`run_experiment` / `log_experiment` / `export_dashboard`); anything that moved since the baseline gets a `harness_edit` row in the ledger — including hand edits, which no tool hook could see. The edit step's verdict discipline stays advisory; the fact of a change is mechanical. +- **Pause switch**: `/autoresearch:off` sets `autoresearchOff: true`, which silences Stop continuation, memory injection and the SessionStart hint; `init_experiment`/`clear_experiments` and the resume command clear it. +- **Memory injection**: UserPromptSubmit injects an aggregated summary (progress + deduped tried directions + best trajectory + discard reasons with rollback rationale + ASI distillation) so progress survives compaction; SessionStart injects the resume hint. Both stay silent while `autoresearchOff` is set. +- **Loop continuation**: the Stop hook blocks (`decision:block`) while a loop is unfinished and not paused (zcode platform limit: 3 consecutive windows). - **Iteration hooks**: `.auto/hooks/before.sh` (pre-benchmark) and `after.sh` (post-record) run on every experiment (fail-open, 30s timeout, stdout → `*_steer`). -- **Hook ecosystem**: `skills/autoresearch-hooks` tutorial + 6 ready-to-use examples in `hooks/examples/` (anti-thrash, hypothesis reflection, idea rotator, learnings journal, auto-tag winners, macOS notify); copy one to `.auto/hooks/` and go (parsed with Node, no jq dependency). +- **Hook ecosystem**: `skills/autoresearch-hooks` tutorial + 7 ready-to-use examples in `hooks/examples/before/` and `hooks/examples/after/` (anti-thrash, hypothesis reflection, idea rotator with shallow-streak nudge, hourly self-check, learnings journal, auto-tag winners, macOS notify); copy one into `.auto/hooks/` renamed to `before.sh` (from `before/`) or `after.sh` (from `after/`) — those exact filenames are what the server executes (parsed with Node, no jq dependency). - **Stop-loss**: after `consecutiveFailures` in a row (default 3, configurable in `.auto/config.json`) the plugin hints you to stop. -- **Ledger audit**: `log_experiment` validates invariants before writing (keep must be a real improvement, a discarded real improvement must have failed the guard, event ordering, commit field); violations are rejected; a crashed segment that wasn't rolled back blocks continuation. `auditBypass: true` in `.auto/config.json` explicitly skips it (not recommended). -- **Benchmark drift detection**: `init_experiment` records hashes of measure.sh/checks.sh; `run_experiment` compares them, and a mid-run benchmark change returns a `benchmark_drift` warning (prevents "faking the metric by editing the benchmark"). -- **Secondary-metric constraints** (opt-in): `log_experiment` supports `constraints: [{name, maxPct}]`. On keep, secondary metrics must stay within maxPct% of the first run's value; anything beyond rejects the keep (prevents reward hacking like "trading memory for speed"). +- **Ledger audit**: `log_experiment` validates invariants before writing (keep must be a real improvement, a discarded real improvement must have failed the guard, a checks_failed row must be attested by a failed-checks verdict on the latest run, event ordering, commit field), and every logged metric must equal what `run_experiment` observed — the run's observation snapshot persists in config.json and is consumed by each successful log (one log per run; a crash is only attested when nothing was measured); cross-call `pending*` state is cleared wholesale at segment boundaries (`init_experiment`/`clear_experiments`), so a previous session's observation never attests the next one; violations are rejected; a noop on a dirty working tree carries an explicit warning; a crashed segment that wasn't rolled back blocks continuation. `auditBypass: true` in `.auto/config.json` explicitly skips both the audit and secondary-metric constraints (not recommended). +- **Benchmark drift detection**: `init_experiment` records hashes of measure.sh/checks.sh; `run_experiment` compares them, and a mid-run benchmark change returns a `benchmark_drift` warning (surfaces benchmark edits instead of silently accepting incomparable metrics). +- **Secondary-metric constraints** (opt-in): `log_experiment` supports `constraints: [{name, maxPct}]`. On keep, secondary metrics must stay within maxPct% of the first run's value; anything beyond rejects the keep (a "faster but much heavier" change is not a valid keep). ## Directory structure @@ -76,20 +80,19 @@ plugin/ ├── .mcp.json # MCP stdio server declaration ├── mcp/ │ ├── server.ts # JSON-RPC line protocol + tools -│ └── lib/ # pure logic: experiment / ledger / git / validate / dashboard / dashboard-server / html / paths +│ └── lib/ # pure logic: experiment / ledger / git / validate / dashboard / dashboard-server / html / paths / bash / frozen / harness / types ├── hooks/ -│ ├── hooks.json # Stop / PreToolUse / PermissionRequest / UserPromptSubmit / SessionStart +│ ├── hooks.json # Stop / PreToolUse / UserPromptSubmit / SessionStart │ ├── stop-continue.ts # loop unfinished → block │ ├── guard-frozen.ts # frozen-file write protection → deny -│ ├── permission-gate.ts # experiment-tool gating → deny │ ├── memory-inject.ts # ledger tail injection │ ├── session-start.ts # session resume hints -│ └── examples/ # 6 ready-to-use before/after iteration hooks +│ └── examples/ # 7 ready-to-use before/after iteration hooks ├── skills/ │ ├── autoresearch/ # SKILL.md thin router + references/ │ └── autoresearch-hooks/ # iteration-hook tutorial ├── commands/ # autoresearch / export / off / clear / finalize -├── scripts/finalize.sh # /autoresearch:finalize implementation +├── scripts/ # launcher.mjs (cross-platform Node resolver for server + hooks) / finalize.sh (/autoresearch:finalize) └── tests/ # node --test unit tests ``` @@ -99,20 +102,27 @@ Setting `"workingDir": "work/"` in `.auto/config.json` separates the research di ## Session state (`.auto/`) -| File | Purpose | -| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `log.jsonl` | **append-only single source of truth**: config lines + run lines; segments advance on config lines | -| `prompt.md` | session charter (goal/metric/scope/Off Limits/What's Been Tried) | -| `measure.sh` | benchmark script (frozen) | -| `checks.sh` | optional correctness gate (frozen) | -| `config.json` | optional session overrides: `maxIterations`, `consecutiveFailures`, `workingDir`, `auditBypass`, `autoresearchOff` (server-managed keys like `benchmarkHashes`/`pendingChecksFailed` also live here) | -| `ideas.md` | optional hypothesis list | +| File | Purpose | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `log.jsonl` | **append-only single source of truth**: config lines + run lines; segments advance on config lines; hook / harness_edit / frozen_attempt rows ride along as observations and never affect loop state | +| `goal.md` | the deep goal, recorded verbatim from the user at setup (frozen) | +| `scope.md` | the user-agreed scope boundary (files in scope + off limits, frozen) | +| `prompt.md` | session charter (goal/metric/scope/Off Limits/What's Been Tried) | +| `measure.sh` | benchmark script (frozen) | +| `checks.sh` | optional correctness gate (frozen) | +| `config.json` | optional session overrides: `maxIterations`, `consecutiveFailures`, `workingDir`, `auditBypass`, `autoresearchOff`, loop calibration `plateauWindow` / `plateauMinImprovement` / `deterministic` (server-managed keys like `benchmarkHashes`/`pendingChecksFailed`/`pendingObservation` also live here; `pending*` keys are cross-call state, cleared at segment boundaries) | +| `ideas.md` | optional hypothesis list | +| `experience.md` | optional cross-segment lessons (at most one per segment; survives clear) | + +The manifest `userConfig` (`maxIterations` / timeouts) seeds the **MCP server's** defaults via env; the hooks (Stop continuation, memory injection) cannot see it — they read `.auto/config.json` overrides only and fall back to 20 runs / 3 failures. Set session-level overrides in `config.json` when hooks and server must agree. -## Known limits (research-backed, see `docs/research/autoresearch-survey.md` §4.1) +## Known limits (research-backed, see `docs/research/autoresearch-survey.md` §4.1 at the repository root) -- **Unattended runs**: two layers - the Stop-hook continuation window (3 consecutive) inside a session, plus scheduled self-wakeup via `CronCreate` (recurring + maxRuns, user-authorized only; see the skill's `references/unattended.md`). Infinite auto-resume stays impossible by platform design. +- **Unattended runs**: two layers - the Stop-hook continuation window (3 consecutive) inside a session, plus scheduled self-wakeup via `CronCreate` (finite schedule + `maxRuns`, user-authorized only; see the skill's `references/unattended.md`). Infinite auto-resume stays impossible by platform design. - **Headless mode (`--prompt`) does not run hooks**: guardrails take effect in interactive sessions; run autoresearch in an interactive session. -- `git add -A` commits unrelated dirty files together (known pi inheritance); commit a clean baseline during setup. +- **Frozen-file write protection covers file tools only**: the PreToolUse guard sees `Write`/`Edit`/`ApplyPatch`; shell redirections (e.g. `echo x > .auto/measure.sh`) bypass it and are caught by benchmark drift detection instead — detection after the fact, not prevention. +- **Windows spawn chain verified**: the MCP server and all hooks start via `node scripts/launcher.mjs` (plain `node`, no POSIX shell involved); resolver branches are covered on Windows by `tests/launcher.test.ts` — report anything odd there. +- A keep commits every uncommitted change inside the research directory — unrelated files included (known pi inheritance); commit a clean baseline during setup. Changes outside the research dir (`workingDir` mode) are never committed, and any pre-staged files elsewhere in the repo are unstaged first so the commit describes exactly this experiment. ## Development diff --git a/external_plugins/autoresearch/README_CN.md b/external_plugins/autoresearch/README_CN.md index f0729220..f9ecddd7 100644 --- a/external_plugins/autoresearch/README_CN.md +++ b/external_plugins/autoresearch/README_CN.md @@ -4,7 +4,7 @@ 让 ZCode 的 coding agent 在**固定度量**上自主迭代优化：改代码 → 跑基准 → 保留改进、回滚退化 → 循环。 -基于 [karpathy/autoresearch](https://github.com/karpathy/autoresearch) 与 [pi-autoresearch](https://github.com/davebcn87/pi-autoresearch) 的调研（见仓库根 `docs/research/autoresearch-survey.md`）。架构决策见 `adr/decisions/1-*.md`、`2-*.md`。 +基于 [karpathy/autoresearch](https://github.com/karpathy/autoresearch) 与 [pi-autoresearch](https://github.com/davebcn87/pi-autoresearch) 的调研（见仓库根 `docs/research/autoresearch-survey.md`）。架构决策见仓库根 `adr/implemented/`。 ## 安装 @@ -12,61 +12,65 @@ 1. 添加市场：本地目录 / 本仓库地址。 2. 在 **Settings → Plugin Management** 安装并启用 `autoresearch`。 -3. 插件提供：MCP 服务（5 个工具）、skill `autoresearch`、5 个命令、5 个 hook。启用插件即授予代码执行信任（官方约定）。 +3. 插件提供：MCP 服务（5 个工具）、2 个 skill（`autoresearch` + `autoresearch-hooks`）、5 个命令、4 个 hook。启用插件即授予代码执行信任（官方约定）。 > 无第三方 npm 依赖：MCP server 与 hooks 均为 Node 标准库 TypeScript 脚本（Node ≥24，类型由 Node 原生剥离，无构建步骤）。 +> +> 运行要求：机器上需存在 Node ≥24，但不要求它在宿主 PATH 上。MCP server 与 hooks 经 `scripts/launcher.mjs` 以裸 `node` 启动（`node` 是宿主在三平台都能解析的唯一命令），解析顺序 PATH → nvm（POSIX `$NVM_DIR` / Windows `%NVM_HOME%` 或 `%APPDATA% +vm`，最新优先）→ Homebrew 或 `/usr/local/bin`（POSIX）→ `%ProgramFiles% +odejs`（Windows），可用 `AUTORESEARCH_NODE_BIN` 显式指定（`node scripts/launcher.mjs --resolve` 可查看解析结果）。基准脚本（`.auto/measure.sh`、`.auto/checks.sh`）仍是 bash，Windows 上需要 Git-Bash 或 WSL。 ## 安全与副作用 启用本插件即授予代码执行信任（官方市场约定）。插件会： - **执行命令**：`run_experiment` 运行你编写的基准脚本 `.auto/measure.sh`，以及存在时的正确性门禁 `.auto/checks.sh`； -- **自动执行 git 操作**：keep 时自动 `git commit`，非 keep 时自动回滚（`.auto/` 豁免回滚）； -- **安装 ZCode hooks**：Stop（循环续跑）、PreToolUse（冻结文件写保护）、PermissionRequest（实验工具门禁）、UserPromptSubmit/SessionStart（账本记忆注入）； -- **启动本地 HTTP dashboard**：`export_dashboard` 监听 127.0.0.1； +- **自动执行 git 操作**：keep 时自动 `git commit`，discard/crash/checks_failed 时自动回滚（noop 无改动可回；会话状态 `.auto/`、宿主会话目录 `.zcode/` 与 `autoresearch-dashboard.html` 不参与 commit/回滚）； +- **安装 ZCode hooks**：Stop（循环续跑）、PreToolUse（冻结文件写保护）、UserPromptSubmit（账本记忆注入）、SessionStart（续跑提示）； +- **启动本地 HTTP dashboard**：`export_dashboard` 监听 127.0.0.1，编辑部式档案页呈现全部实验历史（跨 segment 趋势、亮暗双主题）； - **写入会话状态**：项目目录下的 `.auto/`（`log.jsonl`、`config.json`）。 ## 用法 ``` /autoresearch:autoresearch <目标> # 进入/恢复 autoresearch 模式（无会话则走 setup） -/autoresearch:export # 导出静态 dashboard（autoresearch-dashboard.html） -/autoresearch:off # 暂停循环续跑（autoresearchOff: true） +/autoresearch:export # 导出 dashboard（live SSE 地址 + 静态 autoresearch-dashboard.html） +/autoresearch:off # 暂停循环与其提示（autoresearchOff: true） /autoresearch:clear # 重置会话账本 /autoresearch:finalize # 把 kept 实验整理为干净分支（scripts/finalize.sh） ``` 或让 skill 自动触发（描述含 "autoresearch"、"自主优化" 等）。一次完整循环： -1. **Setup**：定机械度量 → 写 `.auto/measure.sh`（输出 `METRIC name=value` 行）→ 可选 `.auto/checks.sh`（正确性门禁）→ 写 `.auto/prompt.md` 章程 → 建实验分支 `git checkout -b autoresearch/`。 +1. **Setup**：定机械度量 → 写 `.auto/measure.sh`（输出 `METRIC name=value` 行）→ 可选 `.auto/checks.sh`（正确性门禁）→ 逐字记录用户深层目标到 `.auto/goal.md` 与用户同意的范围边界到 `.auto/scope.md`（均冻结，hook 拒写）→ 写 `.auto/prompt.md` 章程（目标/范围段逐字引用两文件）→ 建实验分支 `git checkout -b autoresearch/`。 2. `init_experiment`（metric_name、direction）→ 跑一次 baseline。 -3. **循环**：一次聚焦改动 → `run_experiment` → `log_experiment`（keep 自动 commit / 非 keep 自动回滚，`.auto/` 豁免）。 +3. **循环**：一次聚焦改动 → `run_experiment` → `log_experiment`（keep 自动 commit；discard/crash/checks_failed 自动回滚；noop 无改动可回；`.auto/`、宿主会话目录 `.zcode/` 与 dashboard 均不计入）。 ## 工具（MCP） -| 工具 | 作用 | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `init_experiment` | 建立/重启实验 segment（名称、主度量、方向 lower/higher） | -| `run_experiment` | 跑基准：计时、`METRIC name=value` 解析、10 行/4KB 截断回传、超时杀进程组、`repeat` 取中位数、执行 `before.sh` 钩子 | -| `log_experiment` | 记录结果：keep 自动 `git commit`（`experiment:` 前缀）；非 keep 自动回滚（豁免 `.auto/`）；返回 baseline/best/delta/confidence/plateau 与下一步提示；segment 内存在带 rollback 理由的 discard 时返回 `revisit_nudge` 重访提示（有意重试时以 `asi.revisits_run: ` 标注）；本次 confidence 同时快照写入账本 run 行；执行 `after.sh` 钩子 | -| `export_dashboard` | 起本地 live dashboard（127.0.0.1 + SSE 自动刷新）并写静态 HTML 兜底 | -| `clear_experiments` | 删除 `.auto/log.jsonl` 重置会话（保留 measure/checks/prompt） | +| 工具 | 作用 | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init_experiment` | 建立/重启实验 segment（名称、主度量、方向 lower/higher）；可选循环校准：`plateau_min_improvement` / `plateau_window`（贴近指标真实步长设置，避免平台期误报）与 `deterministic`（确定性基准下提示不再建议 repeat 复测） | +| `run_experiment` | 跑基准：`.auto/measure.sh` 存在时 `command` 可缺省（默认执行它），自由模式仍必填；计时、`METRIC name=value` 解析、10 行/4KB 截断回传、超时杀进程组、`repeat` 取中位数、执行 `before.sh` 钩子 | +| `log_experiment` | 记录结果：keep 自动 `git commit`（`experiment:` 前缀）；discard/crash/checks_failed 自动回滚（noop 无改动可回；豁免会话状态 `.auto/`、`.zcode/` 与 dashboard）；以 `rolled_back`（restored_to 哈希 + 文件清单）显式回报 discard/crash 回滚；返回 baseline/best/delta/confidence（单位 × noise）/plateau/`runs_left` 与下一步提示；discard/checks_failed 落账且 segment 内存在带 rollback 理由的 discard 时返回 `revisit_nudge` 重访提示（有意重试时以 `asi.revisits_run: ` 标注，环境导致的 crash 以 `asi.infra: true` 标注，infra 行不计入 doom-loop 信号）；本次 confidence 同时快照写入账本 run 行；执行 `after.sh` 钩子 | +| `export_dashboard` | 起本地 live dashboard（127.0.0.1 + SSE 自动刷新）并写静态 HTML 兜底 | +| `clear_experiments` | 删除 `.auto/log.jsonl` 重置会话（其余 `.auto/` 文件全部保留） | ## 护栏 -- **benchmark 锁定**：`.auto/measure.sh` 存在时，`run_experiment` 只执行该脚本（剥 env/time/nice 包装后校验）。 -- **正确性背压**：`.auto/checks.sh` 存在时 benchmark 通过后自动执行；失败禁 keep 并自动回滚。 -- **写保护**：PreToolUse hook deny 对 `.auto/measure.sh` / `.auto/checks.sh` 的写入。 -- **工具门禁（近似）**：无会话时 PermissionRequest hook deny 实验工具（仅覆盖经权限询问的调用）。 -- **自动激活提示**：SessionStart 检测活动会话时注入续跑引导；`/autoresearch:off` 可暂停（`autoresearchOff: true`）。 -- **记忆注入**：UserPromptSubmit/SessionStart hook 注入聚合摘要（进度 + 已尝试方向去重 + best 轨迹 + 弃用方向与理由 + ASI 提炼），compaction 后不丢进度；检测到重复/震荡尝试（doom-loop）时提示换方向。 -- **循环续跑**：Stop hook 在循环未结束时 `decision:block`（zcode 平台限制连续 3 次窗口）。 +- **基准锁定**：`.auto/measure.sh` 存在时，`run_experiment` 只执行该脚本（剥 env/time/nice 包装与前导赋值后校验；赋值前缀的值同走字符白名单，夹带 `$(`、反引号等元字符即拒绝）。 +- **正确性背压**：`.auto/checks.sh` 存在时 benchmark 通过后自动执行；失败禁 keep 并自动回滚。从未产生判定（不可执行、被信号杀死、退出码 ≥126）属基础设施故障而非失败——不否决 keep，以 `verdict: false` 如实呈现。 +- **写保护**：PreToolUse hook deny 对 `.auto/measure.sh` / `.auto/checks.sh` / `.auto/goal.md` / `.auto/scope.md` 的 Write/Edit/ApplyPatch 改写（ApplyPatch 的目标从 patch_text 解析）（setup 首次创建是预期例外，冻结指「记录后不可改写」；goal.md 是深层目标原文，循环所服务的根不可变，harness 编辑步只修订其表层表示；scope.md 是用户同意的范围边界，循环在栏内工作，扩围/收围由用户手改或新 segment）。被 deny 的改写企图以 `frozen_attempt` 行记入账本。 +- **harness 留痕**：可编辑面（`prompt.md`、`ideas.md`、`experience.md`、`.auto/hooks/before.sh` / `after.sh`）在记账时点（`run_experiment` / `log_experiment` / `export_dashboard`）做哈希对账，相对基线有变化的文件各记一条 `harness_edit` 行，连手改也逃不掉。编辑步的 verdict 纪律仍是 advisory，但「动过」这件事是机械保证。 +- **暂停开关**：`/autoresearch:off` 写入 `autoresearchOff: true`，同时静音 Stop 续跑、记忆注入与 SessionStart 提示；`init_experiment`/`clear_experiments` 与 resume 命令会清除标记。 +- **记忆注入**：UserPromptSubmit 注入聚合摘要（进度 + 已尝试方向去重 + best 轨迹 + 弃用方向与理由 + ASI 提炼），compaction 后不丢进度；SessionStart 注入续跑提示；`autoresearchOff` 时两者都静默。 +- **循环续跑**：Stop hook 在循环未结束且未暂停时 `decision:block`（zcode 平台限制连续 3 次窗口）。 - **迭代钩子**：`.auto/hooks/before.sh`（基准前）与 `after.sh`（记录后）每次实验自动执行（fail-open，30s 超时，stdout→`*_steer`）。 -- **钩子生态**：`skills/autoresearch-hooks` 教学 + `hooks/examples/` 6 个现成示例（防重复失败/换思路/假设反思/学习日志/通知/最优打标），复制到 `.auto/hooks/` 即用（node 解析，无 jq 依赖）。 +- **钩子生态**：`skills/autoresearch-hooks` 教学 + `hooks/examples/before/` 与 `hooks/examples/after/` 共 7 个现成示例（防重复失败/换思路+浅层连击提醒/假设反思/时段自检/学习日志/通知/最优打标），复制到 `.auto/hooks/` 并改名为 `before.sh`（before/ 下的示例）或 `after.sh`（after/ 下的示例）——server 只执行这两个固定文件名（node 解析，无 jq 依赖）。 - **止损**：连续失败达 `.auto/config.json` 的 `consecutiveFailures`（默认 3，可配）时提示停止。 -- **账本审计**：`log_experiment` 写入前校验不变量（keep 必须真实改进、discard 真改进须 failed guard、事件顺序、commit 字段），违规拒收；crash 未回滚禁止续跑。`.auto/config.json` 的 `auditBypass: true` 可显式跳过（不推荐）。 -- **基准漂移检测**：`init_experiment` 记录 measure.sh/checks.sh 哈希，`run_experiment` 比对；基准中途变更时返回 `benchmark_drift` 警告（防"改基准造假 metric"）。 -- **次级度量约束**（opt-in）：`log_experiment` 支持 `constraints: [{name, maxPct}]`。keep 时校验次级度量不超首轮值的 maxPct%，超界拒收（防"用内存换速度"类 reward hacking）。 +- **账本审计**：`log_experiment` 写入前校验不变量（keep 必须真实改进、discard 真改进须 failed guard、checks_failed 须有最近一次 run 的失败判定背书、事件顺序、commit 字段），且落账 metric 必须等于 `run_experiment` 的观测值——run 的观测快照持久化于 config.json、每次成功 log 消费（一次 run 对应一次 log；crash 仅在未观测到度量时有据；`pending*` 跨调用态在 segment 边界——`init_experiment`/`clear_experiments`——整体清除，上一 session 的观测不为本 session 背书），违规拒收；脏树上的 noop 携带显式警告；crash 未回滚禁止续跑。`.auto/config.json` 的 `auditBypass: true` 会同时跳过审计与次级度量约束（不推荐）。 +- **基准漂移检测**：`init_experiment` 记录 measure.sh/checks.sh 哈希，`run_experiment` 比对；基准中途变更时返回 `benchmark_drift` 警告（暴露基准改动，不静默接受不可比 metric）。 +- **次级度量约束**（opt-in）：`log_experiment` 支持 `constraints: [{name, maxPct}]`。keep 时校验次级度量不超首轮值的 maxPct%，超界拒收（"更快但重得多"的 keep 不合法）。 ## 目录结构 @@ -76,20 +80,19 @@ plugin/ ├── .mcp.json # MCP stdio server 声明 ├── mcp/ │ ├── server.ts # JSON-RPC 换行协议 + 5 个工具 -│ └── lib/ # 纯逻辑：experiment / ledger / git / validate / dashboard / dashboard-server / html / paths +│ └── lib/ # 纯逻辑：experiment / ledger / git / validate / dashboard / dashboard-server / html / paths / bash / frozen / harness / types ├── hooks/ -│ ├── hooks.json # Stop / PreToolUse / PermissionRequest / UserPromptSubmit / SessionStart +│ ├── hooks.json # Stop / PreToolUse / UserPromptSubmit / SessionStart │ ├── stop-continue.ts # 循环未结束 → block │ ├── guard-frozen.ts # 冻结文件写保护 → deny -│ ├── permission-gate.ts # 实验工具门禁 → deny │ ├── memory-inject.ts # 账本尾行注入 │ ├── session-start.ts # 会话恢复提示 -│ └── examples/ # 6 个现成的 before/after 迭代钩子示例 +│ └── examples/ # 7 个现成的 before/after 迭代钩子示例 ├── skills/ │ ├── autoresearch/ # SKILL.md 薄路由 + references/ │ └── autoresearch-hooks/ # 迭代钩子教学 ├── commands/ # autoresearch / export / off / clear / finalize -├── scripts/finalize.sh # /autoresearch:finalize 实现 +├── scripts/ # launcher.mjs（server 与 hooks 的跨平台 Node 解析入口）/ finalize.sh（/autoresearch:finalize 实现） └── tests/ # node --test 单元测试 ``` @@ -99,20 +102,27 @@ plugin/ ## 会话状态（`.auto/`） -| 文件 | 作用 | -| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `log.jsonl` | **append-only 单一事实源**：config 行 + run 行；segment 由 config 行推进 | -| `prompt.md` | 会话章程（目标/度量/范围/Off Limits/What's Been Tried） | -| `measure.sh` | 基准脚本（冻结） | -| `checks.sh` | 可选正确性门禁（冻结） | -| `config.json` | 可选会话覆盖项：`maxIterations`、`consecutiveFailures`、`workingDir`、`auditBypass`、`autoresearchOff`（`benchmarkHashes`/`pendingChecksFailed` 等服务端管理键也存于此） | -| `ideas.md` | 可选假设清单 | +| 文件 | 作用 | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `log.jsonl` | **append-only 单一事实源**：config 行 + run 行；segment 由 config 行推进；hook / harness_edit / frozen_attempt 行是随行的观测记录，不影响循环状态 | +| `goal.md` | 深层目标，setup 时逐字记录自用户（冻结） | +| `scope.md` | 用户同意的范围边界（范围内文件 + 禁改项，冻结） | +| `prompt.md` | 会话章程（目标/度量/范围/Off Limits/What's Been Tried） | +| `measure.sh` | 基准脚本（冻结） | +| `checks.sh` | 可选正确性门禁（冻结） | +| `config.json` | 可选会话覆盖项：`maxIterations`、`consecutiveFailures`、`workingDir`、`auditBypass`、`autoresearchOff`、循环校准 `plateauWindow`/`plateauMinImprovement`/`deterministic`（`benchmarkHashes`/`pendingChecksFailed`/`pendingObservation` 等服务端管理键也存于此；`pending*` 前缀为跨调用态，segment 边界整体清除） | +| `ideas.md` | 可选假设清单 | +| `experience.md` | 可选跨 segment 经验（每段至多一条；clear 后保留） | -## 已知边界（研究实证，详见报告 §4.1） +清单 `userConfig`（`maxIterations` / 超时）经 env 注入，只作为 **MCP server 侧**的默认值；hooks（Stop 续跑、记忆注入）看不到它——只读 `.auto/config.json` 覆盖项，缺省按 20 轮 / 3 连败。需要 hooks 与 server 口径一致时，在 `config.json` 里设会话级覆盖。 -- **无人值守**：双层形态——会话内靠 Stop hook 续跑窗口（连续 3 次），跨 turn 靠 `CronCreate` 有界定时自唤醒（recurring + maxRuns，须经用户明确授权；见技能 `references/unattended.md`）。无限 auto-resume 仍是平台级不可行。 +## 已知边界（研究实证，详见报告 `docs/research/autoresearch-survey.md` §4.1） + +- **无人值守**：双层形态。会话内靠 Stop hook 续跑窗口（连续 3 次），跨 turn 靠 `CronCreate` 有界定时自唤醒（有限日程 + `maxRuns`，须经用户明确授权；见技能 `references/unattended.md`）。无限 auto-resume 仍是平台级不可行。 - **无头模式（`--prompt`）不执行 hooks**：护栏在交互式会话生效；请用交互式会话跑 autoresearch。 -- `git add -A` 会把无关脏文件一起 commit（继承 pi 的已知弱点）；setup 时先提交干净基线。 +- **冻结文件写保护只覆盖文件类工具**：PreToolUse 守卫只看 `Write`/`Edit`/`ApplyPatch`；shell 重定向（如 `echo x > .auto/measure.sh`）绕过它，由基准漂移检测事后捕获——是检测而非阻止。 +- **Windows 启动链已验证**：MCP server 与全部 hooks 经 `node scripts/launcher.mjs` 启动（裸 `node`，不经 POSIX sh）；解析分支在 Windows 上由 `tests/launcher.test.ts` 覆盖——遇到异常请反馈。 +- keep 会提交研究目录内所有未提交改动——含无关文件（继承 pi 的已知弱点）；setup 时先提交干净基线。`workingDir` 模式下研究目录之外的改动不会被提交，仓库内其他位置预暂存的文件会先被 unstage，保证 commit 恰好描述本次实验。 ## 开发 diff --git a/external_plugins/autoresearch/commands/autoresearch.md b/external_plugins/autoresearch/commands/autoresearch.md index 85ba4720..c7ad09b4 100644 --- a/external_plugins/autoresearch/commands/autoresearch.md +++ b/external_plugins/autoresearch/commands/autoresearch.md @@ -1,14 +1,14 @@ --- -description: Enter autoresearch mode. Resumes from .auto/prompt.md when a session exists, otherwise sets one up from your goal. Usage: /autoresearch:autoresearch +description: Enter autoresearch mode. Resumes when .auto/log.jsonl exists (the ledger is the session), otherwise sets one up from your goal. Usage: /autoresearch:autoresearch --- Enter autoresearch mode for this workspace. -1. If `.auto/log.jsonl` and `.auto/prompt.md` exist, **resume**: read the charter and the ledger, then continue the loop (one focused change → `run_experiment` → `log_experiment`). +1. If `.auto/log.jsonl` exists, **resume** (the tools and hooks treat the ledger as the session — do not re-run setup: the frozen files already exist and the hook will deny rewriting them): first clear `autoresearchOff` if it is set (set it to false in `.auto/config.json`), then read the charter (`.auto/prompt.md`) and the ledger and continue the loop (one focused change → `run_experiment` → `log_experiment`). If `.auto/prompt.md` is missing, reconstruct a minimal charter from the ledger's config rows and the frozen goal/scope files before continuing. 2. Otherwise **set up a new session** from the goal in $ARGUMENTS: - Load the skill `autoresearch` (or read `skills/autoresearch/SKILL.md`). - - Follow `references/setup-guide.md`: pick a mechanical metric, create `.auto/measure.sh` (prints `METRIC name=value`), optional `.auto/checks.sh`, write `.auto/prompt.md` charter. + - Follow `references/setup-guide.md`: pick a mechanical metric, create `.auto/measure.sh` (prints `METRIC name=value`), optional `.auto/checks.sh`, record the user's deep goal verbatim in `.auto/goal.md` and the agreed scope boundary in `.auto/scope.md`, write `.auto/prompt.md` charter (its goal/scope sections quote the two files verbatim). Re-targeting after a `/autoresearch:clear`: the old frozen files still exist and the guard denies their rewrites — remove them first (user-approved) when the new target needs a different benchmark or gate. - `init_experiment` with metric name and direction, run a baseline, then start the loop. -3. Remind the user: keep the benchmark frozen (`.auto/measure.sh` / `.auto/checks.sh` are write-protected by the plugin hook), and use `/autoresearch:export` for a dashboard. +3. Remind the user: keep the session anchors frozen (`.auto/measure.sh` / `.auto/checks.sh` / `.auto/goal.md` / `.auto/scope.md` are write-protected by the plugin hook once created), and use `/autoresearch:export` for a dashboard. $ARGUMENTS diff --git a/external_plugins/autoresearch/commands/clear.md b/external_plugins/autoresearch/commands/clear.md index a322c66d..2ea9ab73 100644 --- a/external_plugins/autoresearch/commands/clear.md +++ b/external_plugins/autoresearch/commands/clear.md @@ -1,12 +1,12 @@ --- -description: Clear the autoresearch session by deleting .auto/log.jsonl, then start fresh. Keeps measure.sh / checks.sh / prompt.md. Usage: /autoresearch:clear +description: Clear the autoresearch session by deleting .auto/log.jsonl, then start fresh. All other .auto/ files are kept. Usage: /autoresearch:clear --- Clear the current autoresearch session. -1. Confirm with the user that they want to wipe the experiment history. This cannot be undone: the session state is gone, though the ledger and all `experiment:` commits remain in git history. +1. Confirm with the user that they want to wipe the experiment history. This cannot be undone: the ledger is deleted, while all `experiment:` commits remain in git history. 2. Call the `clear_experiments` tool. 3. **Scheduled wakeups**: run `CronList` and `CronDelete` every remaining `[autoresearch wake]` task of this session (scheduled tasks cannot be deleted from automation turns, so this must happen in a user turn). -4. Report the result. A fresh target can now start with `/autoresearch:autoresearch ` or `init_experiment`. +4. Report the result. A fresh target can now start with `/autoresearch:autoresearch ` or `init_experiment`. If the new target needs a different benchmark or correctness gate, the old frozen files (`.auto/measure.sh`, `.auto/checks.sh`, and the old `goal.md`/`scope.md`) must be removed first — clear keeps them and the write guard still denies rewrites of files that exist (user-deleted, or an explicitly user-approved command). Note: kept `experiment:` commits remain in git history; only the `.auto/` session ledger is reset. diff --git a/external_plugins/autoresearch/commands/export.md b/external_plugins/autoresearch/commands/export.md index e88a61fa..d3f8e647 100644 --- a/external_plugins/autoresearch/commands/export.md +++ b/external_plugins/autoresearch/commands/export.md @@ -4,8 +4,8 @@ description: Render .auto/log.jsonl into autoresearch-dashboard.html. Usage: /au Export the autoresearch experiment dashboard. -1. Call the `export_dashboard` tool (prefer the MCP tool; the same export logic also lives in `${ZCODE_PLUGIN_ROOT}/mcp/server.ts`). +1. Call the `export_dashboard` tool (prefer the MCP tool; the same export logic also lives in `${ZCODE_PLUGIN_ROOT}/mcp/server.ts`). Besides writing the static file, it starts a live SSE dashboard on `127.0.0.1` (auto-refreshing) and returns its `url`. 2. If the tool is unavailable, fall back to reading `.auto/log.jsonl` yourself, summarizing experiments (status, metric, delta vs baseline, direction), and writing a self-contained `autoresearch-dashboard.html` in the workspace root. -3. Tell the user the file path (`autoresearch-dashboard.html`) and a 2-3 line summary of progress (experiments run, kept, best metric). +3. Tell the user the live `url` (open it in a browser) and the static file path (`autoresearch-dashboard.html`), plus a 2-3 line summary of progress (experiments run, kept, best metric). If there is no `.auto/log.jsonl`, say so and suggest `/autoresearch:autoresearch` to start a session first. diff --git a/external_plugins/autoresearch/commands/finalize.md b/external_plugins/autoresearch/commands/finalize.md index da1c7484..8305db0a 100644 --- a/external_plugins/autoresearch/commands/finalize.md +++ b/external_plugins/autoresearch/commands/finalize.md @@ -5,7 +5,7 @@ description: Split kept experiments into clean topic branches you can PR. Usage: Finalize the experiment session into clean, PR-able topic branches. 1. Read `.auto/log.jsonl`, collect the **kept** experiments (status=keep with a commit). -2. Group them by file dependency: two experiments may share a branch only if their changed files overlap; group small, keep order. +2. Group experiments that touch the same files into the same branch; separate branches must have disjoint file sets (the script rejects overlap). Keep groups small and in commit order. The script verifies that the union of the group branches equals **every** change between the trunk merge-base and HEAD (session files excluded) — manual, non-experiment commits included — so either fold ungrouped commits into a group or the verify step fails and rolls everything back. 3. Write `groups.json` at the project root: ```json { @@ -24,6 +24,7 @@ Finalize the experiment session into clean, PR-able topic branches. `last_commit` must be the full kept commit hash (`git rev-parse `). 4. Run `bash ${ZCODE_PLUGIN_ROOT}/scripts/finalize.sh `. 5. **Scheduled wakeups**: run `CronList` and `CronDelete` every remaining `[autoresearch wake]` task of this session (scheduled tasks cannot be deleted from automation turns, so this must happen in a user turn). -6. Report: the created branches (`autoresearch//NN-`), the overall metric improvement, and cleanup notes (`git branch -D` + `rm -r .auto` when done). +6. **Experience review (before any cleanup)**: if `.auto/experience.md` exists, read it with the user and fold the surviving lessons into the project's `AGENTS.md` (drop `NO_ENTRY` stubs and lessons the user rejects). This step MUST complete before deleting the session directory — `experience.md` only lives in `.auto/` and is lost otherwise. +7. Report: the created branches (`autoresearch//NN-`), the overall metric improvement, and cleanup notes (`git branch -D` + `rm -r .auto`) — cleanup runs **only after** the experience review above is done. If the script reports a file appearing in multiple groups, merge those groups or re-split and rerun. diff --git a/external_plugins/autoresearch/commands/off.md b/external_plugins/autoresearch/commands/off.md index d0a23d74..5d666443 100644 --- a/external_plugins/autoresearch/commands/off.md +++ b/external_plugins/autoresearch/commands/off.md @@ -1,11 +1,11 @@ --- -description: Stop the auto-resume hints while keeping the session. Resume anytime with /autoresearch:autoresearch. Usage: /autoresearch:off +description: Pause the loop and its hints while keeping the session. Resume anytime with /autoresearch:autoresearch. Usage: /autoresearch:off --- Pause autoresearch without wiping the session. -1. Set `autoresearchOff: true` in `.auto/config.json` (create the file if missing). The SessionStart hook will stop injecting "resume" hints for this workspace. +1. Set `autoresearchOff: true` in `.auto/config.json` (create the file if missing). Edit the single key — do not rewrite the whole file: server-managed keys (`benchmarkHashes`, `harnessHashes`, `pending*`) live here, and a full-file rewrite silently disarms drift detection and the harness trace. Stop continuation, memory injection and the SessionStart resume hint all stop for this workspace. 2. The ledger and all experiment commits stay intact. 3. **Scheduled wakeups**: if unattended mode was active, run `CronList` and `CronDelete` every remaining `[autoresearch wake]` task of this session (scheduled tasks cannot be deleted from automation turns, so this must happen in a user turn; until deleted or maxRuns is hit, leftover wakes idle with one cheap reply). -4. To resume: run `/autoresearch:autoresearch` (it ignores the off marker), or clear the marker (`autoresearchOff: false`) for hints again. +4. To resume: run `/autoresearch:autoresearch` (it clears the off marker), or set `autoresearchOff: false` manually. 5. To start completely fresh: `/autoresearch:clear`. diff --git a/external_plugins/autoresearch/hooks/examples/after/macos-notify.sh b/external_plugins/autoresearch/hooks/examples/after/macos-notify.sh index 37d68fab..afe82fe1 100755 --- a/external_plugins/autoresearch/hooks/examples/after/macos-notify.sh +++ b/external_plugins/autoresearch/hooks/examples/after/macos-notify.sh @@ -13,8 +13,13 @@ const run = p.run_entry; const session = p.session; const title = `autoresearch run ${run.run}: ${run.status}`; const body = `metric=${run.metric ?? '—'} (best=${session?.best_metric ?? '—'}) ${run.description ?? ''}`; +// AppleScript string literal: escape backslashes FIRST (else the quote escape +// itself gets escaped), then quotes; fold newlines — a literal newline inside +// an AppleScript string is a syntax error. +const esc = (s) => + s.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\r?\n/g, ' '); try { - execFileSync('osascript', ['-e', `display notification "${body.replace(/"/g, '\\"')}" with title "${title.replace(/"/g, '\\"')}"`], { stdio: 'ignore' }); + execFileSync('osascript', ['-e', `display notification "${esc(body)}" with title "${esc(title)}"`], { stdio: 'ignore' }); } catch { /* no osascript (non-macOS) → silent */ } diff --git a/external_plugins/autoresearch/hooks/examples/before/anti-thrash.sh b/external_plugins/autoresearch/hooks/examples/before/anti-thrash.sh index bdd3bcf8..04e1b131 100755 --- a/external_plugins/autoresearch/hooks/examples/before/anti-thrash.sh +++ b/external_plugins/autoresearch/hooks/examples/before/anti-thrash.sh @@ -6,7 +6,6 @@ set -euo pipefail readonly STREAK_THRESHOLD=3 -readonly WINDOW=5 payload="$(cat)" @@ -21,14 +20,16 @@ const tail = fs.readFileSync(log, 'utf8') .filter(e => e && e.type === 'run') .slice(-5); +// Consecutive-failure semantics match the guardrails spec: only real failures +// count; keep AND noop both break the chain. let streak = 0; for (const r of [...tail].reverse()) { - if (r.status === 'keep') break; + if (r.status === 'keep' || r.status === 'noop') break; streak += 1; } if (streak < 3) process.exit(0); -console.log(`⚠️ ${streak} consecutive non-keep results. Consider:`); +console.log(`⚠️ ${streak} consecutive failed results. Consider:`); console.log(' - re-reading .auto/prompt.md and the benchmark script'); console.log(' - something structurally different, not another variation of the same idea'); console.log(' - measuring where time/space is actually spent before the next change'); diff --git a/external_plugins/autoresearch/hooks/examples/before/idea-rotator.sh b/external_plugins/autoresearch/hooks/examples/before/idea-rotator.sh index c848b72b..780d2ffc 100755 --- a/external_plugins/autoresearch/hooks/examples/before/idea-rotator.sh +++ b/external_plugins/autoresearch/hooks/examples/before/idea-rotator.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash # idea-rotator: pick the next untried idea from .auto/ideas.md (one idea per # line, lines starting with '#' are ignored) and steer the agent to try it. +# Also checks the ledger for a shallow streak: when the recent tagged +# hypotheses (asi.hypothesis tier prefixes) are all [tune]/[ensemble] with no +# [struct], nudge the agent back to the ideation list. # before hook. Silent when there is no ideas file or no untried ideas left. set -euo pipefail @@ -10,17 +13,43 @@ node - "$payload" <<'NODE' const p = JSON.parse(process.argv[2]); const fs = require('fs'); const ideasFile = `${p.cwd}/.auto/ideas.md`; -if (!fs.existsSync(ideasFile)) process.exit(0); +if (fs.existsSync(ideasFile)) { + const lines = fs.readFileSync(ideasFile, 'utf8') + .split('\n') + .map(l => l.trim()) + .filter(l => l && !l.startsWith('#')); -const lines = fs.readFileSync(ideasFile, 'utf8') - .split('\n') - .map(l => l.trim()) - .filter(l => l && !l.startsWith('#')); + if (lines.length > 0) { + // Rotate using the run counter so each experiment surfaces a different idea. + const idx = (p.session?.run_count ?? 0) % lines.length; + console.log(`💡 untried idea to consider: ${lines[idx]}`); + console.log(' (add/remove lines in .auto/ideas.md to control the pool)'); + } +} -if (lines.length === 0) process.exit(0); - -// Rotate using the run counter so each experiment surfaces a different idea. -const idx = (p.session?.run_count ?? 0) % lines.length; -console.log(`💡 untried idea to consider: ${lines[idx]}`); -console.log(' (add/remove lines in .auto/ideas.md to control the pool)'); +// Shallow-streak check over the ledger's last (up to 6) run rows. +// Only tagged hypotheses count; untagged ones stay invisible (backward +// compatible with older ledgers). Advisory: this is a nudge, not a gate. +try { + const ledger = `${p.cwd}/.auto/log.jsonl`; + if (!fs.existsSync(ledger)) process.exit(0); + const rows = fs.readFileSync(ledger, 'utf8') + .split('\n') + .filter(l => l.trim()) + .map(l => { try { return JSON.parse(l); } catch { return null; } }) + .filter(e => e && e.type === 'run') + .slice(-6); + const tagged = rows + .map(r => String(r?.asi?.hypothesis || '')) + .filter(h => /^\[(struct|tune|ensemble)\]/.test(h)); + if (tagged.length < 3) process.exit(0); + const shallow = tagged.filter(h => !h.startsWith('[struct]')); + if (shallow.length === tagged.length) { + console.log(' ⚠️ shallow streak: the recent tagged hypotheses are all [tune]/[ensemble].'); + console.log(' Rebalance before burning more runs: pick or invent one [struct] direction'); + console.log(' (see the cold-start list in .auto/ideas.md).'); + } +} catch { + // fail open: never block the loop over ledger-reading trouble +} NODE diff --git a/external_plugins/autoresearch/hooks/examples/before/self-check.sh b/external_plugins/autoresearch/hooks/examples/before/self-check.sh new file mode 100755 index 00000000..69eabc42 --- /dev/null +++ b/external_plugins/autoresearch/hooks/examples/before/self-check.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# self-check: wall-clock self-assessment for long sessions. When more than +# CHECK_INTERVAL seconds have passed since the last self-check (tracked in +# .auto/selfcheck.state, created on demand), inject a short diagnostic +# checklist — am I still doing structural work, or drifting into shallow +# tuning? Most useful in unattended/overnight runs; harmless otherwise. +# before hook. Silent when the interval has not elapsed or the session has +# no started_at epoch (older ledgers). +set -euo pipefail + +payload="$(cat)" + +CHECK_INTERVAL="${AUTORESEARCH_SELF_CHECK_INTERVAL:-3600}" + +node - "$payload" "$CHECK_INTERVAL" <<'NODE' +const p = JSON.parse(process.argv[2]); +const interval = Number(process.argv[3]) || 3600; +const fs = require('fs'); + +const started = p.session?.started_at_epoch; +if (!started) process.exit(0); // older ledger: no segment clock, stay silent + +const stateFile = `${p.cwd}/.auto/selfcheck.state`; +let last = 0; +try { last = Number(fs.readFileSync(stateFile, 'utf8').trim()) || 0; } catch {} + +const now = Math.floor(Date.now() / 1000); +if (now - last < interval) process.exit(0); + +try { fs.writeFileSync(stateFile, String(now)); } catch {} + +console.log('🕐 hourly self-check — answer these briefly before the next run:'); +console.log(' 1. Is my current direction structural work, or shallow tuning in disguise?'); +console.log(' 2. Did recent failures get a root-cause analysis, or did I just reach for the safest next move?'); +console.log(' 3. Is there an insight in the ledger (a near-miss, an unexplained delta) I have not acted on?'); +console.log(' 4. Given the time left: is my frontrunner truly my best candidate, or just the most familiar one?'); +console.log(' If any answer points at drift, course-correct now (cold-start list in .auto/ideas.md).'); +NODE diff --git a/external_plugins/autoresearch/hooks/guard-frozen.ts b/external_plugins/autoresearch/hooks/guard-frozen.ts index 61abb91f..a25d8ec6 100644 --- a/external_plugins/autoresearch/hooks/guard-frozen.ts +++ b/external_plugins/autoresearch/hooks/guard-frozen.ts @@ -1,17 +1,36 @@ #!/usr/bin/env node -// PreToolUse hook: deny writes to the frozen benchmark scripts -// (.auto/measure.sh, .auto/checks.sh). The matcher limits this to -// Write|Edit|ApplyPatch; path filtering happens here, per zcode docs. +// PreToolUse hook: deny writes to frozen session files — the benchmark +// scripts (.auto/measure.sh, .auto/checks.sh), the deep-goal record +// (.auto/goal.md) and the user-agreed scope boundary (.auto/scope.md). +// The matcher limits this to Write|Edit|ApplyPatch; path filtering happens +// here, per zcode docs. Write/Edit carry file_path/path; ApplyPatch carries +// only patch_text, so its per-file target lines are parsed here — otherwise +// the patch tool would bypass the guard entirely (evidence 2026-11-04: +// ApplyPatch on a frozen file exited 0 with no deny and no frozen_attempt). import { resolve, relative } from "node:path"; +import { existsSync } from "node:fs"; import { resolveWorkCwd } from "../mcp/lib/paths.ts"; +import { FROZEN_FILES, FROZEN_REASONS, frozenKey } from "../mcp/lib/frozen.ts"; +import { appendLedgerEntry } from "../mcp/lib/ledger.ts"; interface PreToolUseInput { - tool_input?: { file_path?: string; path?: string }; - toolInput?: { file_path?: string; path?: string }; + tool_name?: string; + tool_input?: { file_path?: string; path?: string; patch_text?: string }; + toolInput?: { file_path?: string; path?: string; patch_text?: string }; } -const cwd = resolveWorkCwd(process.argv[2] || process.cwd()); -const FROZEN = new Set([".auto/measure.sh", ".auto/checks.sh"]); +const projectCwd = process.argv[2] || process.cwd(); +let cwd: string; +try { + cwd = resolveWorkCwd(projectCwd); +} catch (err) { + // Hook discipline: fail open, but never silently — an invalid workingDir is + // a misconfiguration the user must see (stderr is the diagnostic channel). + process.stderr.write( + `[autoresearch] ${err instanceof Error ? err.message : String(err)}\n`, + ); + cwd = projectCwd; +} let raw = ""; process.stdin.setEncoding("utf8"); @@ -25,23 +44,74 @@ try { } const ti = input.tool_input || input.toolInput || {}; -const fp = ti.file_path || ti.path || ""; -if (!fp) process.exit(0); -let rel: string; -try { - rel = relative(cwd, resolve(cwd, fp)); -} catch { - process.exit(0); +// Target paths of an ApplyPatch payload: the *** File: headers plus the +// *** Move to: destination. Best-effort by design — an unparseable patch falls +// through to "no candidates" and is allowed (fail-open discipline unchanged). +function patchTargets(patchText: string): string[] { + const out: string[] = []; + for (const line of patchText.split(/\r?\n/)) { + const m = + /^\*\*\* (?:Update|Add|Delete|Move) File: (.+)$/.exec(line) ?? + /^\*\*\* Move to: (.+)$/.exec(line); + if (m) out.push(m[1].trim()); + } + return out; +} + +const direct = ti.file_path || ti.path; +const candidates: string[] = direct + ? [String(direct)] + : typeof ti.patch_text === "string" + ? patchTargets(ti.patch_text) + : []; +if (candidates.length === 0) process.exit(0); + +// Frozen means "immutable once recorded", not "never written": setup creates +// these files (goal/scope verbatim from the user) before any segment exists. +// What the guard blocks is rewriting a record already on disk. +const hits: string[] = []; +for (const fp of candidates) { + let rel: string; + try { + rel = frozenKey(relative(cwd, resolve(cwd, fp))); + } catch { + continue; + } + if (!FROZEN_FILES.has(rel)) continue; + if (!existsSync(resolve(cwd, rel))) continue; + if (!hits.includes(rel)) hits.push(rel); +} +if (hits.length === 0) process.exit(0); + +// The attempt itself is a fact: a denied rewrite of a frozen record goes to +// the ledger, so goalpost-moving shows up in the archive instead of vanishing +// with the denied tool call. Bookkeeping must never gate the guard — a failed +// ledger write still denies. +let traced = true; +for (const rel of hits) { + try { + appendLedgerEntry(cwd, { + type: "frozen_attempt", + ts: new Date().toISOString(), + file: rel, + tool: input.tool_name ?? null, + }); + } catch { + traced = false; + } } -if (!FROZEN.has(rel)) process.exit(0); process.stdout.write( JSON.stringify({ hookSpecificOutput: { hookEventName: "PreToolUse", permissionDecision: "deny", - permissionDecisionReason: `[autoresearch] ${rel} is frozen: the benchmark metric must not change during the loop. If you really need a new metric, start over: init_experiment with a new target.`, + permissionDecisionReason: `[autoresearch] ${hits.join(", ")} ${ + hits.length === 1 ? "is" : "are" + } frozen: ${hits + .map((rel) => FROZEN_REASONS[rel] ?? "frozen session file.") + .join("; ")}${traced ? " (attempt recorded to the ledger.)" : ""}`, }, }), ); diff --git a/external_plugins/autoresearch/hooks/hooks.json b/external_plugins/autoresearch/hooks/hooks.json index a6bf2c40..98286069 100644 --- a/external_plugins/autoresearch/hooks/hooks.json +++ b/external_plugins/autoresearch/hooks/hooks.json @@ -8,6 +8,7 @@ "type": "process", "command": "node", "args": [ + "${ZCODE_PLUGIN_ROOT}/scripts/launcher.mjs", "${ZCODE_PLUGIN_ROOT}/hooks/stop-continue.ts", "${ZCODE_PROJECT_DIR}" ], @@ -25,6 +26,7 @@ "type": "process", "command": "node", "args": [ + "${ZCODE_PLUGIN_ROOT}/scripts/launcher.mjs", "${ZCODE_PLUGIN_ROOT}/hooks/guard-frozen.ts", "${ZCODE_PROJECT_DIR}" ], @@ -34,22 +36,6 @@ ] } ], - "PermissionRequest": [ - { - "hooks": [ - { - "type": "process", - "command": "node", - "args": [ - "${ZCODE_PLUGIN_ROOT}/hooks/permission-gate.ts", - "${ZCODE_PROJECT_DIR}" - ], - "timeoutMs": 5000, - "statusMessage": "autoresearch: gating experiment tools…" - } - ] - } - ], "UserPromptSubmit": [ { "hooks": [ @@ -57,6 +43,7 @@ "type": "process", "command": "node", "args": [ + "${ZCODE_PLUGIN_ROOT}/scripts/launcher.mjs", "${ZCODE_PLUGIN_ROOT}/hooks/memory-inject.ts", "${ZCODE_PROJECT_DIR}" ], @@ -68,12 +55,13 @@ ], "SessionStart": [ { - "matcher": "startup|resume", + "matcher": "startup|resume|compact", "hooks": [ { "type": "process", "command": "node", "args": [ + "${ZCODE_PLUGIN_ROOT}/scripts/launcher.mjs", "${ZCODE_PLUGIN_ROOT}/hooks/session-start.ts", "${ZCODE_PROJECT_DIR}" ], diff --git a/external_plugins/autoresearch/hooks/memory-inject.ts b/external_plugins/autoresearch/hooks/memory-inject.ts index 2e634e28..7f290f12 100644 --- a/external_plugins/autoresearch/hooks/memory-inject.ts +++ b/external_plugins/autoresearch/hooks/memory-inject.ts @@ -2,7 +2,11 @@ // UserPromptSubmit hook: inject an aggregated session-memory summary so the // loop survives compaction and long sessions — progress, deduplicated tried // directions, best trajectory, recent runs with ASI, and a doom-loop warning. -import { rebuildState, readSessionConfig } from "../mcp/lib/ledger.ts"; +import { + rebuildState, + readSessionConfig, + plateauCalibration, +} from "../mcp/lib/ledger.ts"; import { resolveWorkCwd } from "../mcp/lib/paths.ts"; import { directionLabel, @@ -13,7 +17,17 @@ import { import type { SessionState } from "../mcp/lib/types.ts"; const projectCwd = process.argv[2] || process.cwd(); -const cwd = resolveWorkCwd(projectCwd); +let cwd: string; +try { + cwd = resolveWorkCwd(projectCwd); +} catch (err) { + // Invalid workingDir: fail open with a diagnostic (hook discipline), the + // MCP server surfaces the same misconfiguration as a hard error. + process.stderr.write( + `[autoresearch] ${err instanceof Error ? err.message : String(err)}\n`, + ); + cwd = projectCwd; +} function pass(): never { process.exit(0); @@ -22,9 +36,12 @@ function pass(): never { let state: SessionState | undefined; try { const cfg = readSessionConfig(projectCwd); + if (cfg.autoresearchOff === true) pass(); const max = Number(cfg.maxIterations); + const calib = plateauCalibration(projectCwd); state = rebuildState(cwd, { maxIterations: Number.isFinite(max) && max > 0 ? max : 20, + deterministic: calib.deterministic, }); } catch { pass(); @@ -40,7 +57,8 @@ lines.push( `已跑 ${state.runs.length}/${state.maxIterations ?? 20} 次，baseline=${state.baseline ?? "—"}，best=${state.best ?? "—"}。`, ); -// Deduplicated tried directions (similarity-based, most recent label kept) +// Deduplicated tried directions (similarity-based; for each direction the +// first-seen label is kept — later paraphrases collapse into it) const tried: string[] = []; const triedNorm: string[] = []; for (const r of state.runs) { @@ -54,27 +72,28 @@ const triedList = tried.slice(-8); if (triedList.length > 0) lines.push(`已尝试方向：${triedList.join("、")}（避免重复尝试）。`); -// Best trajectory: baseline → improving keeps (≤6 steps) +// Best trajectory: baseline → improving keeps (≤6 steps). Labels reuse +// directionLabel's own 40-char cap — an inner 14-char slice would cut CJK +// conclusions mid-word ("spw 细扫 1" hiding "…单峰确认在 2.0"). const kept = state.runs.filter((r) => r.status === "keep" && r.metric != null); const steps = kept.filter((r) => r.metric !== state.baseline).slice(-6); if (steps.length > 0) { - const traj = steps.map( - (r) => `${r.metric}(${directionLabel(r).slice(0, 14)})`, - ); + const traj = steps.map((r) => `${r.metric}(${directionLabel(r)})`); lines.push(`best 轨迹：${state.baseline ?? "—"} → ${traj.join(" → ")}。`); } // Discard reasons (design D5): why each rejected direction was dropped, so a // later result can be checked against them even outside the 3-run recent // window. Same window as the tried-directions list, one line, reasons -// truncated to keep the injection slim. +// truncated at 120 chars — enough for a full conclusion ("spw 1.5/2.5 细扫 +// 单峰确认在 2.0"), still bounded (8 × 120) to keep the injection slim. const discardItems: string[] = []; for (const r of state.runs) { if (r.status !== "discard" && r.status !== "checks_failed") continue; const raw = r.asi?.rollback; if (typeof raw !== "string" || raw.trim() === "") continue; const reason = raw.trim(); - const truncated = reason.length > 60 ? reason.slice(0, 60) + "…" : reason; + const truncated = reason.length > 120 ? reason.slice(0, 120) + "…" : reason; discardItems.push(`#${r.run} ${truncated}`); } const discardList = discardItems.slice(-8); @@ -108,9 +127,21 @@ if (doom) { ); } -lines.push( - "如果你在运行 autoresearch 循环，请基于上述进度选择下一个假设并继续 run_experiment → log_experiment。", -); +// Continuation guidance must agree with the loop's own terminal states: at +// the cap the server refuses run_experiment (and the Stop hook stands down), +// so a continuation nudge here would push a run that cannot happen — worst +// right after a compaction, when this inject is the agent's main anchor. +const capReached = + state.maxIterations != null && state.runs.length >= state.maxIterations; +if (capReached) { + lines.push( + `迭代上限已到（${state.runs.length}/${state.maxIterations}）：本 segment 请收官总结，勿继续 run_experiment；续优用 init_experiment 开新 segment，暂停用 /autoresearch:off。`, + ); +} else { + lines.push( + "如果你在运行 autoresearch 循环，请基于上述进度选择下一个假设并继续 run_experiment → log_experiment。", + ); +} process.stdout.write( JSON.stringify({ diff --git a/external_plugins/autoresearch/hooks/permission-gate.ts b/external_plugins/autoresearch/hooks/permission-gate.ts deleted file mode 100644 index 595170c3..00000000 --- a/external_plugins/autoresearch/hooks/permission-gate.ts +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env node -// PermissionRequest hook: approximate tool gate (pi-gap M3, #1). -// When the workspace has no active experiment session (.auto/log.jsonl), deny -// permission prompts for the experiment tools so the loop cannot be started by -// accident. With a session, allow. This is an approximation — calls that are -// auto-approved never reach PermissionRequest; tool-internal checks and the -// skill remain the backstop. -import { existsSync } from "node:fs"; -import { join } from "node:path"; -import { resolveWorkCwd } from "../mcp/lib/paths.ts"; - -interface PermissionRequestInput { - tool_name?: string; - toolName?: string; -} - -const projectCwd = process.argv[2] || process.cwd(); -const cwd = resolveWorkCwd(projectCwd); -const EXPERIMENT_TOOLS = new Set([ - "init_experiment", - "run_experiment", - "log_experiment", - "export_dashboard", - "clear_experiments", -]); - -let raw = ""; -process.stdin.setEncoding("utf8"); -for await (const chunk of process.stdin as AsyncIterable) raw += chunk; - -let input: PermissionRequestInput = {}; -try { - input = raw.trim() ? (JSON.parse(raw) as PermissionRequestInput) : {}; -} catch { - process.exit(0); // fail open -} - -const tool = input.tool_name || input.toolName || ""; -if (!EXPERIMENT_TOOLS.has(tool)) process.exit(0); - -const hasSession = existsSync(join(cwd, ".auto", "log.jsonl")); -if (hasSession) process.exit(0); - -process.stdout.write( - JSON.stringify({ - hookSpecificOutput: { - hookEventName: "PermissionRequest", - decision: { - behavior: "deny", - message: `[autoresearch] 当前工作区没有实验会话（.auto/log.jsonl 不存在），${tool} 已被拦截。请先通过 /autoresearch:autoresearch 建立会话。`, - }, - }, - }), -); diff --git a/external_plugins/autoresearch/hooks/session-start.ts b/external_plugins/autoresearch/hooks/session-start.ts index 96c7feef..038cf823 100644 --- a/external_plugins/autoresearch/hooks/session-start.ts +++ b/external_plugins/autoresearch/hooks/session-start.ts @@ -8,7 +8,17 @@ import { join } from "node:path"; import { resolveWorkCwd } from "../mcp/lib/paths.ts"; const projectCwd = process.argv[2] || process.cwd(); -const cwd = resolveWorkCwd(projectCwd); +let cwd: string; +try { + cwd = resolveWorkCwd(projectCwd); +} catch (err) { + // Invalid workingDir: fail open with a diagnostic (hook discipline), the + // MCP server surfaces the same misconfiguration as a hard error. + process.stderr.write( + `[autoresearch] ${err instanceof Error ? err.message : String(err)}\n`, + ); + cwd = projectCwd; +} const log = join(cwd, ".auto", "log.jsonl"); if (!existsSync(log)) process.exit(0); diff --git a/external_plugins/autoresearch/hooks/stop-continue.ts b/external_plugins/autoresearch/hooks/stop-continue.ts index bc03d3a6..80147eba 100644 --- a/external_plugins/autoresearch/hooks/stop-continue.ts +++ b/external_plugins/autoresearch/hooks/stop-continue.ts @@ -2,77 +2,130 @@ // Stop hook: keep the autoresearch loop running while it is not finished. // zcode grants at most 3 consecutive Stop continuations per window, so this // hook only fires when the ledger shows the loop should continue. -import { rebuildState, readSessionConfig } from "../mcp/lib/ledger.ts"; +import { + rebuildState, + readSessionConfig, + plateauCalibration, +} from "../mcp/lib/ledger.ts"; import { resolveWorkCwd } from "../mcp/lib/paths.ts"; import { isStopReached, detectDoomLoop } from "../mcp/lib/experiment.ts"; import type { SessionState } from "../mcp/lib/types.ts"; const projectCwd = process.argv[2] || process.cwd(); -const cwd = resolveWorkCwd(projectCwd); +let cwd: string; +try { + cwd = resolveWorkCwd(projectCwd); +} catch (err) { + // Invalid workingDir: fail open with a diagnostic (hook discipline), the + // MCP server surfaces the same misconfiguration as a hard error. + process.stderr.write( + `[autoresearch] ${err instanceof Error ? err.message : String(err)}\n`, + ); + cwd = projectCwd; +} function failOpen(): never { process.exit(0); } let state: SessionState | undefined; +let calib: ReturnType = { + minImprovement: undefined, + window: undefined, + deterministic: false, +}; +// Whether the continuation window below runs on the built-in default (no +// maxIterations in .auto/config.json): manifest userConfig is invisible to +// hooks, so a server-side cap set that way can be tighter than this window — +// the block reason says so once, so the model can align instead of spinning. +let usedDefaultMax = true; try { const cfg = readSessionConfig(projectCwd); + if (cfg.autoresearchOff === true) failOpen(); const max = Number(cfg.maxIterations); + usedDefaultMax = !(Number.isFinite(max) && max > 0); const fails = Number(cfg.consecutiveFailures); + calib = plateauCalibration(projectCwd); state = rebuildState(cwd, { maxIterations: Number.isFinite(max) && max > 0 ? max : 20, consecutiveFailures: Number.isFinite(fails) && fails > 0 ? fails : 3, + plateauWindow: calib.window, + plateauMinImprovement: calib.minImprovement, + deterministic: calib.deterministic, }); } catch { failOpen(); } // No active session → let the model finish normally. -if (!state || !state.config || state.runs.length === 0) failOpen(); - -const finished = isStopReached( - state.runs, - state.maxIterations ?? 20, - state.failureThreshold, -); -if (finished) failOpen(); +// Everything below the config/rebuild try/catch is fail-open too: a broken +// ledger or an unexpected throw must degrade to "allow the stop" (exit 0), +// never to a crash exit code with a stack trace as the hook's only output. +try { + if (!state || !state.config || state.runs.length === 0) failOpen(); -// Plateau convergence: recent runs improved < 1% → let the model wrap up. -// The Stop schema has no allow-with-reason shape, so the advisory goes to -// stderr (hook log) while the hook exits 0 to allow the stop. -if (state.plateau) { - process.stderr.write( - `[autoresearch] 循环已进入平台期（最近 5 轮改善 < 1%，best=${state.best ?? "—"}）。` + - `建议：用 run_experiment repeat:3 复测确认，或 init_experiment 开启新 segment，或就此收尾总结。\n`, + const finished = isStopReached( + state.runs, + state.maxIterations ?? 20, + state.failureThreshold, ); - process.exit(0); -} else { - const dir = state.config?.direction ?? "lower"; - const tail = state.runs - .slice(-3) - .map((r) => { - let line = `#${r.run} ${r.status} metric=${r.metric ?? "—"} ${r.description ?? ""}`; - if (r.asi && typeof r.asi === "object") { - const parts: string[] = []; - if (r.asi.hypothesis) parts.push(`hyp: ${r.asi.hypothesis}`); - if (r.asi.next_action_hint) - parts.push(`next: ${r.asi.next_action_hint}`); - if (r.asi.rollback) parts.push(`rollback: ${r.asi.rollback}`); - if (parts.length) line += "\n " + parts.join("\n "); - } - return line; - }) - .join("\n"); + if (finished) failOpen(); + + // Plateau convergence: recent runs improved below the calibrated threshold + // (default 1% over 5 runs) → let the model wrap up. The Stop schema has no + // allow-with-reason shape, so the advisory goes to stderr (hook log) while + // the hook exits 0 to allow the stop. + if (state.plateau) { + const window = calib.window ?? 5; + const min = calib.minImprovement ?? 0.01; + const pct = `${(min * 100).toFixed(2).replace(/\.?0+$/, "")}%`; + const suggestions = [ + // A deterministic benchmark cannot be confirmed by re-measurement. + ...(calib.deterministic ? [] : ["用 run_experiment repeat:3 复测确认"]), + "init_experiment 开启新 segment", + "就此收尾总结", + ].join("，或 "); + process.stderr.write( + `[autoresearch] 循环已进入平台期（最近 ${window} 轮改善 < ${pct}，best=${state.best ?? "—"}）。` + + `建议：${suggestions}。\n`, + ); + process.exit(0); + } else { + const dir = state.config?.direction ?? "lower"; + const tail = state.runs + .slice(-3) + .map((r) => { + let line = `#${r.run} ${r.status} metric=${r.metric ?? "—"} ${r.description ?? ""}`; + if (r.asi && typeof r.asi === "object") { + const parts: string[] = []; + if (r.asi.hypothesis) parts.push(`hyp: ${r.asi.hypothesis}`); + if (r.asi.next_action_hint) + parts.push(`next: ${r.asi.next_action_hint}`); + if (r.asi.rollback) parts.push(`rollback: ${r.asi.rollback}`); + if (parts.length) line += "\n " + parts.join("\n "); + } + return line; + }) + .join("\n"); - const reason = - `[autoresearch] 实验循环未结束：segment ${state.segment} 已跑 ${state.runs.length}/${state.maxIterations ?? 20} 次，` + - `direction=${dir}，baseline=${state.baseline ?? "—"}，best=${state.best ?? "—"}。` + - `最近记录：\n${tail}\n` + - (detectDoomLoop(state.runs) - ? `⚠️ 检测到重复/震荡尝试：请停止重复同一假设，换一个结构性不同的方向。\n` - : "") + - `请继续下一个假设：修改代码 → run_experiment → log_experiment（keep/discard）。\n` + - `（若用户已要求无人值守续跑，可按 unattended 规程用 CronCreate 创建有界定时唤醒。）`; + const reason = + `[autoresearch] 实验循环未结束：segment ${state.segment} 已跑 ${state.runs.length}/${state.maxIterations ?? 20} 次，` + + `direction=${dir}，baseline=${state.baseline ?? "—"}，best=${state.best ?? "—"}。` + + `最近记录：\n${tail}\n` + + (detectDoomLoop(state.runs) + ? `⚠️ 检测到重复/震荡尝试：请停止重复同一假设，换一个结构性不同的方向。\n` + : "") + + `请继续下一个假设：修改代码 → run_experiment → log_experiment（keep/discard）。\n` + + (usedDefaultMax + ? `（提示：本续跑窗口按缺省 ${state.maxIterations ?? 20} 次计算；清单 userConfig 与 AR_MAX_ITERATIONS 环境变量的 maxIterations 对 hook 不可见，若 server 侧窗口更小会在 run_experiment 处报「iteration cap reached」——需对齐时在 .auto/config.json 设置 maxIterations。）\n` + : "") + + `（若用户已要求无人值守续跑，可按 unattended 规程用 CronCreate 创建有界定时唤醒。）`; - process.stdout.write(JSON.stringify({ decision: "block", reason })); + process.stdout.write(JSON.stringify({ decision: "block", reason })); + } +} catch (err) { + process.stderr.write( + `[autoresearch] stop-continue 检查失败，放行本次停止: ${err instanceof Error ? err.message : String(err)}\n`, + ); + failOpen(); } diff --git a/external_plugins/autoresearch/mcp/lib/bash.ts b/external_plugins/autoresearch/mcp/lib/bash.ts new file mode 100644 index 00000000..7738a9d3 --- /dev/null +++ b/external_plugins/autoresearch/mcp/lib/bash.ts @@ -0,0 +1,180 @@ +// Bash invocation details shared by the two spawn sites in server.ts: +// shell-argument quoting and Windows process semantics. +// +// Two facts make a naive `spawn("bash", …)` wrong on Windows: +// 1. Two unrelated bash.exe ship on a typical Windows box — Git Bash's +// usr\bin\bash.exe and the WSL launcher at System32\bash.exe. Which one +// PATH yields depends on the caller's environment (an MCP server started +// from a GUI gets a different PATH than a terminal), so commands can +// silently execute in WSL with different path semantics. +// 2. runCommand kills benchmarks via negative-pid process groups +// (process.kill(-pid)); Windows has no such addressing — every call +// fails with ESRCH and the tree survives the timeout, hanging the tool +// call until the benchmark finishes naturally. +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { dirname } from "node:path"; + +let cachedBash: string | null = null; + +// WSL's launcher masquerades as bash.exe in three places: the system copies +// (System32/SysWOW64) and the store's app-execution aliases — machine-wide +// (C:\Windows\WindowsApps) and per-user (%LOCALAPPDATA%\Microsoft\WindowsApps, +// on every default Windows user PATH). None accept Git-Bash-style scripts: +// handing one a repo path fails with "/bin/bash: : No such file or +// directory" (exit 127), and a benchmark command would silently run under +// WSL path semantics instead. +function isWslOrStub(path: string): boolean { + return /(\\Windows\\(System32|SysWOW64|WindowsApps)|\\Microsoft\\WindowsApps)\\/i.test( + path, + ); +} + +// The selection rule for where.exe hits, as a pure function so the stub +// filter and the usr\bin preference are testable on every platform. +export function pickUsableBash(paths: string[]): string | null { + const usable = paths.filter((p) => !isWslOrStub(p)); + // Git for Windows installs bash at \usr\bin\bash.exe — prefer it so + // the outer shell matches what measure.sh authors expect and so + // unixToolsPathEntry sees a coreutils-bearing directory. + const gitBash = usable.find((p) => /\\usr\\bin\\bash\.exe$/i.test(p)); + return gitBash ?? usable[0] ?? null; +} + +// Standard install locations a slim GUI PATH can hide — the bash twin of +// resolveGit's GIT_CANDIDATES. Git for Windows' installer puts bash.exe under +// \usr\bin and \bin, but the default PATH entry it adds is only +// Git\cmd (git's shim directory, no bash), so a GUI-launched server's +// where.exe finds nothing but the WSL launchers. Real evidence 2026-10-29 +// (nano-jev V1 resume): registry PATH carried Git\cmd only; every +// run_experiment died with "no usable bash.exe on PATH". Prefer usr\bin. +const BASH_CANDIDATES = [ + "C:\\Program Files\\Git\\usr\\bin\\bash.exe", + "C:\\Program Files\\Git\\bin\\bash.exe", + "C:\\Program Files (x86)\\Git\\usr\\bin\\bash.exe", + "C:\\Program Files (x86)\\Git\\bin\\bash.exe", +]; + +export function resolveBash(candidates?: string[]): string { + if (candidates !== undefined) { + // Injected candidates (tests) bypass the PATH probe and the cache. + for (const path of candidates) { + if (existsSync(path)) return path; + } + throw new Error( + "no usable bash.exe among injected candidates or standard install locations (install Git for Windows; the WSL launchers in System32 and the WindowsApps aliases are excluded)", + ); + } + if (process.platform !== "win32") return "bash"; + if (cachedBash) return cachedBash; + let hits: string[] = []; + try { + // where.exe reads the Windows PATH directly, no shell in the middle. + hits = execFileSync("where.exe", ["bash"], { encoding: "utf8" }) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + } catch { + hits = []; + } + const resolved = pickUsableBash(hits); + if (resolved) { + cachedBash = resolved; + return resolved; + } + // PATH hides bash entirely — derive it from where git landed (the one Git + // directory the installer does put on PATH) before falling back to the + // installer defaults. \cmd\git.exe ⇒ \{usr\bin,bin}\bash.exe. + try { + const gitHit = execFileSync("where.exe", ["git"], { encoding: "utf8" }) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)[0]; + if (gitHit) { + const root = dirname(dirname(gitHit)); + const derived = [ + `${root}\\usr\\bin\\bash.exe`, + `${root}\\bin\\bash.exe`, + ].find((p) => existsSync(p)); + if (derived) { + cachedBash = derived; + return derived; + } + } + } catch { + // no git on PATH either — fall through to the standard locations + } + for (const path of BASH_CANDIDATES) { + if (existsSync(path)) { + cachedBash = path; + return path; + } + } + throw new Error( + "no usable bash.exe on PATH or standard install locations (install Git for Windows; the WSL launchers in System32 and the WindowsApps aliases are excluded)", + ); +} + +// Single-quote a value for `bash -c`. Paths reach runCommand inside a shell +// string, where an unquoted apostrophe (e.g. "Alex's project") ends the +// quoting early and the command dies with a syntax error; the checks gate +// would read that nonzero exit as "checks failed". Inside single quotes bash +// treats every character literally, so only the quote itself needs escaping. +export function quoteShellArg(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +// Windows equivalent of killing the process group: taskkill /T walks the +// child tree, /F forces termination. Returns false when the tree was +// already gone. On POSIX callers keep their own group-kill semantics. +export function killTree(pid: number): boolean { + if (process.platform !== "win32") return false; + try { + execFileSync("taskkill", ["/pid", String(pid), "/T", "/F"], { + stdio: "ignore", + }); + return true; + } catch { + return false; + } +} + +// Kill the wrapper when the tree walk fails. A failed taskkill means the +// child tree outlives the timeout and the caller waits for an "exit" that may +// never come; killing the wrapper is the lesser failure. +export function killTreeOrWrapper( + pid: number | undefined, + killWrapper: () => void, + treeKiller: (pid: number) => boolean = killTree, +): void { + if (pid == null || treeKiller(pid)) return; + try { + killWrapper(); + } catch { + /* already gone */ + } +} + +// Git for Windows keeps its coreutils (head, grep, date…) next to bash.exe +// in \usr\bin; prepend that directory to spawned shells so benchmark +// and hook scripts keep coreutils. No-op under plain PATH semantics — +// usr\bin must already be on the server's PATH for resolveBash to land +// there — kept as cheap defense for hosts that curate child environments. +export function unixToolsPathEntry(bashPath: string): string | null { + if (process.platform !== "win32") return null; + const dir = dirname(bashPath); + return /usr[\\/]bin$/i.test(dir) ? dir : null; +} + +export function spawnEnv( + bashPath: string, + base: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const entry = unixToolsPathEntry(bashPath); + if (!entry) return base; + // Windows env keys are case-insensitive in spirit but plain objects are + // not; reuse the existing key so we never emit both PATH and Path. + const pathKey = + Object.keys(base).find((key) => key.toLowerCase() === "path") ?? "PATH"; + return { ...base, [pathKey]: `${entry};${base[pathKey] ?? ""}` }; +} diff --git a/external_plugins/autoresearch/mcp/lib/dashboard-server.ts b/external_plugins/autoresearch/mcp/lib/dashboard-server.ts index 3f1ac2ae..476a0ef8 100644 --- a/external_plugins/autoresearch/mcp/lib/dashboard-server.ts +++ b/external_plugins/autoresearch/mcp/lib/dashboard-server.ts @@ -1,15 +1,35 @@ // Local HTTP + SSE dashboard server, hosted inside the MCP server process. // Routes: / (live HTML), /autoresearch.jsonl (ledger raw), /events (SSE). import { createServer } from "node:http"; -import type { Server, ServerResponse } from "node:http"; +import type { IncomingMessage, Server, ServerResponse } from "node:http"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { rebuildState, readSessionConfig } from "./ledger.ts"; +import { readLedger, rebuildHistory } from "./ledger.ts"; import { renderLiveDashboard } from "./dashboard.ts"; +import { safeShortHash } from "./git.ts"; const clients = new Set(); let server: Server | null = null; let boundPort: number | null = null; +let boundWorkCwd: string | null = null; + +function stopServer(): void { + if (!server) return; + for (const res of clients) { + clients.delete(res); + try { + res.destroy(); + } catch { + /* already gone */ + } + } + // close() waits for idle connections; the ephemeral-port rebind below has + // no conflict with it, so the callback is not awaited. + server.close(); + server = null; + boundPort = null; + boundWorkCwd = null; +} function broadcast(): void { // The MCP server process hosts this HTTP server: a single dead client must @@ -29,12 +49,21 @@ function broadcast(): void { function start(workCwd: string): Promise<{ port: number; url: string }> { if (server) { - return Promise.resolve({ - port: boundPort ?? 0, - url: `http://127.0.0.1:${boundPort ?? 0}`, - }); + if (boundWorkCwd === workCwd) { + return Promise.resolve({ + port: boundPort ?? 0, + url: `http://127.0.0.1:${boundPort ?? 0}`, + }); + } + // The research dir moved mid-process (.auto/config.json workingDir edit): + // restart so the live page follows instead of serving the old ledger. + stopServer(); } - const srv = createServer((req, res) => { + const handle = (req: IncomingMessage, res: ServerResponse): void => { + // A disconnecting client can turn any write into an async EPIPE 'error' + // event; swallow it — a dead connection is eviction (req 'close'), never + // a reason for the hosting MCP process to die. + res.on("error", () => {}); const url = new URL(req.url ?? "/", "http://127.0.0.1"); if (url.pathname === "/events") { res.writeHead(200, { @@ -56,22 +85,44 @@ function start(workCwd: string): Promise<{ port: number; url: string }> { res.end("ledger not found (no active session)"); return; } + const ledger = readFileSync(log, "utf8"); res.writeHead(200, { "Content-Type": "application/x-ndjson; charset=utf-8", }); - res.end(readFileSync(log, "utf8")); + res.end(ledger); return; } if (url.pathname === "/" || url.pathname === "") { - const state = rebuildState(workCwd, { - maxIterations: Number(readSessionConfig(workCwd).maxIterations) || 20, + const history = rebuildHistory(readLedger(workCwd)); + const html = renderLiveDashboard(history, { + pin: safeShortHash(workCwd), }); res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); - res.end(renderLiveDashboard(state)); + res.end(html); return; } res.writeHead(404); res.end("not found"); + }; + const srv = createServer((req, res) => { + // A malformed/hand-edited ledger must not take the MCP server process + // down with it: isolate every request and return 500 instead. + try { + handle(req, res); + } catch (err) { + process.stderr.write( + `[autoresearch] dashboard request failed: ${err instanceof Error ? err.message : String(err)}\n`, + ); + try { + if (res.headersSent) res.end(); + else { + res.writeHead(500, { "Content-Type": "text/plain; charset=utf-8" }); + res.end("dashboard request failed"); + } + } catch { + /* client already gone */ + } + } }); return new Promise((resolve, reject) => { srv.on("error", reject); @@ -79,6 +130,7 @@ function start(workCwd: string): Promise<{ port: number; url: string }> { const addr = srv.address(); if (addr && typeof addr === "object") { boundPort = addr.port; + boundWorkCwd = workCwd; server = srv; resolve({ port: boundPort, url: `http://127.0.0.1:${boundPort}` }); } else { @@ -95,10 +147,11 @@ export function ensureDashboardServer(workCwd: string): Promise<{ return start(workCwd); } -export function broadcastDashboardUpdate(): void { - if (server) broadcast(); +/** Shut the live server down (research dir teardown, tests). */ +export function stopDashboardServer(): void { + stopServer(); } -export function dashboardServerInfo(): { url: string } | null { - return server ? { url: `http://127.0.0.1:${boundPort ?? 0}` } : null; +export function broadcastDashboardUpdate(): void { + if (server) broadcast(); } diff --git a/external_plugins/autoresearch/mcp/lib/dashboard.ts b/external_plugins/autoresearch/mcp/lib/dashboard.ts index 56fc7ee1..dfab104c 100644 --- a/external_plugins/autoresearch/mcp/lib/dashboard.ts +++ b/external_plugins/autoresearch/mcp/lib/dashboard.ts @@ -1,14 +1,15 @@ -// Static dashboard renderer: pure function from ledger state to self-contained HTML. +// Archive dashboard renderer: pure function from full ledger history to +// self-contained HTML (editorial layout; static and live share renderBody). import { escapeHtml } from "./html.ts"; -import { deltaFor } from "./ledger.ts"; -import type { LedgerRun, SessionState } from "./types.ts"; +import type { SessionHistory } from "./ledger.ts"; +import type { LedgerRun } from "./types.ts"; -const STATUS_LABEL: Record = { - keep: "keep", - discard: "discard", - crash: "crash", - checks_failed: "checks_failed", - noop: "no-op", +const STATUS_BADGE: Record = { + keep: { label: "KEEP", cls: "keep" }, + discard: { label: "DISCARD", cls: "discard" }, + crash: { label: "CRASH", cls: "crash" }, + checks_failed: { label: "CHECKS", cls: "crash" }, + noop: { label: "NO-OP", cls: "noop" }, }; /** Compact metric number: 4-decimal cap with trailing zeros stripped (42.0000 → 42). */ @@ -21,7 +22,8 @@ function fmtMetric(v: number | null | undefined): string { * Nice grid-tick values for [lo, hi]: step is 1/2/2.5/5 × 10^k, so tick values * stay short and all share the same decimal precision — no 61.6667-style noise. */ -function niceTicks( +/** Exported for tests: tick computation is spec-pinned (decimals invariant). */ +export function niceTicks( lo: number, hi: number, target = 3, @@ -32,8 +34,20 @@ function niceTicks( const step = pow * (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 2.5 ? 2.5 : norm <= 5 ? 5 : 10); - const ds = String(step).split(".")[1]; - const decimals = ds ? ds.replace(/0+$/, "").length : 0; + // String(step) is exponential for sub-micro steps ("2.5e-7"), which a naive + // split(".") misreads ("5e-7" → 4 decimals → every label collapses to + // 0.0000). Below 1, derive decimals from the magnitude instead; the step + // mantissa is a closed set {1, 2, 2.5, 5, 10} × 10^k, and 2.5 carries one + // extra fractional digit. + const exp = Math.floor(Math.log10(step)); + // The +1 tracks the step's mantissa being 2.5 (0.25/2.5e-7 need one extra + // digit), i.e. the same (2, 2.5] band the step selector maps to the 2.5 + // coefficient — not norm === 2.5 exact equality, which misses norm 2.1→step + // 0.25 (gridline labeled "0.3") and norm 2.1e-7→step 2.5e-7 ("0.0000002"). + const decimals = + step >= 1 + ? (String(step).split(".")[1]?.replace(/0+$/, "").length ?? 0) + : Math.min(20, -exp + (norm > 2 && norm <= 2.5 ? 1 : 0)); const out: Array<{ value: number; decimals: number }> = []; const first = Math.ceil((lo - 1e-9) / step); const last = Math.floor((hi + 1e-9) / step); @@ -41,224 +55,488 @@ function niceTicks( return out; } +/** Overall best-vs-baseline change in percent, direction-aware improvement. */ +function overallPct( + first: number | null, + best: number | null, + direction: string, +): { text: string; cls: string } | null { + if (first == null || best == null || !Number.isFinite(first) || first === 0) + return null; + const pct = ((best - first) / Math.abs(first)) * 100; + const improved = direction === "higher" ? pct > 0 : pct < 0; + if (Math.abs(pct) < 1e-9) return { text: "±0%", cls: "neut" }; + const text = `${pct > 0 ? "+" : "−"}${Number(Math.abs(pct).toFixed(1))}%`; + return { text, cls: improved ? "up" : "down" }; +} + /** - * Inline-SVG metric trend: valid metric points in run order (keep filled, - * others hollow), baseline as a dashed reference, light horizontal grid ticks - * with values on the left. Fewer than 2 points → "". Zero external resources — + * Inline-SVG metric trend across the whole ledger: valid metric points in + * global order (keep filled, others hollow, each labelled with its ordinal), + * the first measured metric as a dashed baseline. Fewer than 2 points → + * "". Zero external resources — * the dashboard must stay self-contained. */ -function renderTrendSvg(state: SessionState): string { - const pts = state.runs.filter( - (r): r is LedgerRun & { metric: number } => - r.metric != null && Number.isFinite(r.metric), - ); +function renderTrendSvg( + history: SessionHistory, + variant: "desktop" | "mobile", +): string { + const pts: Array<{ run: LedgerRun; ordinal: number }> = []; + for (const [i, r] of history.runs.entries()) { + if (r.metric != null && Number.isFinite(r.metric)) + pts.push({ run: r, ordinal: i + 1 }); + } if (pts.length < 2) return ""; - const W = 860; - const H = 120; - const padX = 44; // left gutter carries the grid-tick values - const padY = 10; - const vals = pts.map((r) => r.metric); - let lo = Math.min(...vals, state.baseline ?? Infinity); - let hi = Math.max(...vals, state.baseline ?? -Infinity); + // Both variants are emitted; CSS picks one per viewport. The mobile variant + // renders at 1:1 scale (a downscaled 960px SVG makes its labels unreadable). + const dims = + variant === "mobile" + ? { + W: 390, + H: 240, + padX: 30, + padY: 26, + dotR: 3.5, + lineW: 2, + svgCls: "chart-m", + } + : { + W: 960, + H: 240, + padX: 52, // left gutter carries the grid-tick values + padY: 24, + dotR: 4, + lineW: 1.8, + svgCls: "chart-d", + }; + const { W, H, padX, padY, dotR, lineW, svgCls } = dims; + const total = history.totalRuns; + const vals = pts.map((p) => p.run.metric as number); + let lo = Math.min(...vals, history.firstMetric ?? Infinity); + let hi = Math.max(...vals, history.firstMetric ?? -Infinity); if (hi === lo) { lo -= 1; // degenerate flat line → pad it so points sit mid-chart hi += 1; } const span = hi - lo; - const x = (i: number) => padX + (i * (W - 2 * padX)) / (pts.length - 1); + const x = (n: number) => padX + ((n - 1) * (W - 2 * padX)) / (total - 1); const y = (v: number) => padY + ((hi - v) * (H - 2 * padY)) / span; + const tickFont = variant === "mobile" ? 11 : 10; + const labelFont = variant === "mobile" ? 9.5 : 9; + const labelLift = variant === "mobile" ? 12 : 10; // Light horizontal grid ticks at "nice" values covering [lo, hi]. const ticks = niceTicks(lo, hi) .map(({ value: v, decimals }) => { const yy = y(v); return ( - `` + - `${v.toFixed(decimals)}` + `` + + `${v.toFixed(decimals)}` ); }) .join(""); + const baseLine = + history.firstMetric != null && + Number.isFinite(history.firstMetric) && + history.firstMetric >= lo && + history.firstMetric <= hi + ? `` + + `baseline ${escapeHtml(fmtMetric(history.firstMetric))}` + : ""; const poly = pts.map( - (r, i) => `${x(i).toFixed(1)},${y(r.metric).toFixed(1)}`, + (p) => `${x(p.ordinal).toFixed(1)},${y(p.run.metric as number).toFixed(1)}`, ); - const circleCls = (st: string) => - st === "keep" - ? "c-keep" - : st === "discard" - ? "c-discard" - : st === "noop" - ? "c-noop" - : "c-fail"; - const circles = pts - .map( - (r, i) => - ``, - ) + const dots = pts + .map((p) => { + const cls = p.run.status === "keep" ? "dk" : "dh"; // discard / crash / checks_failed / noop → hollow red + return ( + `#${p.ordinal} · ${escapeHtml(fmtMetric(p.run.metric))} · ${escapeHtml(p.run.description ?? "")}` + + `#${p.ordinal}` + ); + }) .join(""); - const baseline = state.baseline; - const baseLine = - baseline != null && - Number.isFinite(baseline) && - baseline >= lo && - baseline <= hi - ? `` + - `baseline ${escapeHtml(fmtMetric(baseline))}` - : ""; - return ` + return ` ${ticks} - ${baseLine} -${circles} + +${dots} `; } -export function renderDashboard(state: SessionState): string { - return renderBody(state, false); +/** Δ text vs the segment baseline; 0 renders neutral (no ▲/▼ mismatch). */ +function deltaFor( + metric: number | null | undefined, + baseline: number | null, + direction: string, +): { text: string; cls: string } | null { + if (metric == null || !Number.isFinite(metric) || baseline == null) + return null; + const raw = metric - baseline; + const delta = direction === "higher" ? raw : -raw; + if (Math.abs(delta) < 1e-9) return { text: "±0", cls: "neut" }; + return delta > 0 + ? { text: `▲ +${fmtMetric(delta)}`, cls: "up" } + : { text: `▼ −${fmtMetric(-delta)}`, cls: "down" }; +} + +export function renderDashboard( + history: SessionHistory, + opts: { pin?: string | null } = {}, +): string { + return renderBody(history, false, opts); } /** Live variant: same body plus an SSE client that auto-reloads on updates. */ -export function renderLiveDashboard(state: SessionState): string { - return renderBody(state, true); +export function renderLiveDashboard( + history: SessionHistory, + opts: { pin?: string | null } = {}, +): string { + return renderBody(history, true, opts); } -function renderBody(state: SessionState, live: boolean): string { - const cfg = state.config; - const direction = cfg?.direction ?? "lower"; - const rows = state.runs - .map((r, i) => { - const delta = deltaFor(state, r.metric); - const deltaText = - delta == null ? "—" : (delta >= 0 ? "+" : "") + fmtMetric(delta); - const improved = delta != null && delta > 0; - const cls = - r.status === "keep" - ? "keep" - : r.status === "discard" - ? "discard" - : r.status === "noop" - ? "noop" - : "crash"; - // Grounded-revisit marker: only a positive integer counts; anything - // else (missing, 0, negative, non-numeric) renders the plain row. - const revisits = r.asi?.revisits_run; - const revisitsN = - typeof revisits === "number" && - Number.isInteger(revisits) && - revisits > 0 - ? revisits - : null; - return { i: i + 1, run: r, deltaText, improved, cls, revisitsN }; - }) - .reverse(); // newest first +function renderBody( + history: SessionHistory, + live: boolean, + opts: { pin?: string | null }, +): string { + const current = history.config; + const direction = current?.direction ?? "lower"; + const name = current?.name ?? "session"; + const pin = opts.pin ?? null; + + // Confidence verdict wording: level → human verdict (colored by level). + const CONF_VERDICT: Record = { + green: "PASS", + yellow: "WEAK", + red: "FAIL", + }; + const conf = history.confidence; + // Result-first hero metrics: the baseline→best story as the centerpiece, + // process counts + confidence verdict demoted to a quiet meta line. + const hasResult = + history.firstMetric != null && + Number.isFinite(history.firstMetric) && + history.bestMetric != null && + Number.isFinite(history.bestMetric); + const overall = hasResult + ? overallPct(history.firstMetric, history.bestMetric, direction) + : null; + const result = hasResult + ? `
+
${fmtMetric(history.firstMetric)}
baseline
+ +
${fmtMetric(history.bestMetric)}
best
+ ${ + overall + ? `
${overall.text}
overall
` + : "" + } +
` + : ""; + const noop = history.runs.filter((r) => r.status === "noop").length; + const metaItems = [ + `${history.totalRuns} experiments`, + `${history.kept} kept`, + `${history.totalRuns - history.kept - noop} reverted`, + `${noop} no-op`, + ]; + if (conf) { + const verdict = CONF_VERDICT[conf.level] ?? escapeHtml(conf.level); + metaItems.push( + `${verdict} ${conf.confidence.toFixed(2)}× noise`, + ); + } + const meta = metaItems.join(` · `); + const heroMetrics = + history.totalRuns === 0 + ? "" + : ` + ${result} +

${meta}

`; + + // Flat ledger: every run in ledger order, no segment grouping. Δ is + // computed against the global baseline (first measured metric). + const rows: string[] = []; + history.runs.forEach((r, i) => { + const badge = STATUS_BADGE[r.status] ?? { + label: escapeHtml(r.status), + cls: "noop", + }; + const d = deltaFor(r.metric, history.firstMetric, direction); + // Grounded-revisit marker: only a positive integer counts; anything + // else (missing, 0, negative, non-numeric) renders the plain row. + const revisits = r.asi?.revisits_run; + const revisitsN = + typeof revisits === "number" && Number.isInteger(revisits) && revisits > 0 + ? revisits + : null; + // ADR-11 (help-first): hypothesis / next_action_hint / rollback carry the + // loop's reasoning but only reached the agent via memory-inject — the + // dashboard showed nothing. Render them as a collapsible block so a human + // reading the trend can answer "why did this round do what it did". + const asiParts: Array<[string, string]> = []; + if (typeof r.asi?.hypothesis === "string" && r.asi.hypothesis.length > 0) + asiParts.push(["假设", r.asi.hypothesis]); + if ( + typeof r.asi?.next_action_hint === "string" && + r.asi.next_action_hint.length > 0 + ) + asiParts.push(["下一步", r.asi.next_action_hint]); + if (typeof r.asi?.rollback === "string" && r.asi.rollback.length > 0) + asiParts.push(["回滚理由", r.asi.rollback]); + const asiBlock = + asiParts.length > 0 + ? `
假设与结论${asiParts + .map( + ([k, v]) => + `
${k}${escapeHtml(v)}
`, + ) + .join("")}
` + : ""; + rows.push(`
+
${i + 1}
+
${badge.label}
+
${fmtMetric(r.metric)}${r.metric != null && current?.metricUnit ? `${escapeHtml(current.metricUnit)}` : ""}
+
${d ? `${d.text}` : `—`}
+
${escapeHtml(r.description ?? "")}${revisitsN != null ? ` ↻ Revisiting #${revisitsN}` : ""}${asiBlock}
+
${r.commit ? `${escapeHtml(r.commit)}` : ""}
+
`); + }); - const kept = state.runs.filter((r) => r.status === "keep"); - const failures = state.runs.filter((r) => r.status !== "keep"); - const conf = state.confidence; - const trend = renderTrendSvg(state); + // Ledger rows are hand-editable: never assume timestamp is a string, and + // escape it like every other piece of ledger-derived text. + const lastRunAt = + typeof history.lastRunAt === "string" ? history.lastRunAt : null; + const lastDate = lastRunAt ? escapeHtml(lastRunAt.slice(0, 10)) : null; + const footer = ` + autoresearch${lastDate ? ` · ${lastDate}` : ""}${history.totalRuns} runs${pin ? ` · pinned at ${escapeHtml(pin)}` : ""}`; + + const trend = + renderTrendSvg(history, "desktop") + renderTrendSvg(history, "mobile"); + const body = + history.totalRuns === 0 + ? `

暂无实验记录。

` + : `${trend ? `
度量趋势 全部实验 · ${escapeHtml(current?.metricName ?? "metric")}● 保留 · ○ 未保留
${trend}
` : ""} +

实验记录 ${history.totalRuns}

+
#状态metricΔ vs baseline假设与结论commit
+${rows.join("\n")} +
`; return ` -autoresearch: ${escapeHtml(cfg?.name ?? "session")} +autoresearch: ${escapeHtml(name)} -

autoresearch · ${escapeHtml(cfg?.name ?? "session")}

-
- ${cfg ? `metric ${escapeHtml(cfg.metricName ?? "")} · direction ${escapeHtml(direction)}` : "no session yet"} - ${cfg?.metricUnit ? ` · unit ${escapeHtml(cfg.metricUnit)}` : ""} -
-
-
${state.runs.length}
experiments
-
${kept.length}
kept
-
${failures.length}
reverted
-
${fmtMetric(state.baseline)}
baseline
-
${fmtMetric(state.best)}
best
- ${conf ? `
${escapeHtml(conf.level)}
confidence ${conf.confidence.toFixed(2)}
` : ""} -
-${trend ? `
${trend}
` : ""} -${ - rows.length === 0 - ? "

暂无实验记录。

" - : ` -
- - - -${rows - .map( - (r) => ` - - - - - - -`, - ) - .join("\n")} - -
#statusmetricΔ vs baselinecommitdescription
${r.i}${STATUS_LABEL[r.run.status] ?? escapeHtml(r.run.status)}${fmtMetric(r.run.metric)}${r.improved ? "▲" : "▼"} ${escapeHtml(r.deltaText)}${r.run.commit ? `${escapeHtml(r.run.commit)}` : "—"}${escapeHtml(r.run.description ?? "")}${r.revisitsN != null ? ` ↻ Revisiting #${r.revisitsN}` : ""}
-
` -} -${ - live - ? `` - : "" -} + : "" + } + `; } diff --git a/external_plugins/autoresearch/mcp/lib/experiment.ts b/external_plugins/autoresearch/mcp/lib/experiment.ts index 06981022..f6a56a75 100644 --- a/external_plugins/autoresearch/mcp/lib/experiment.ts +++ b/external_plugins/autoresearch/mcp/lib/experiment.ts @@ -2,7 +2,10 @@ // No I/O here so they are unit-testable without a workspace. import type { Confidence, Direction, RunLike } from "./types.ts"; -export const METRIC_RE = /^METRIC\s+([\w.µ]+)=(\S+)$/; +// Metric name charset includes `-` (p95-style names) — an over-tight class +// would silently drop the primary metric and keep would then fail with a +// confusing "metric must be a finite number" (worth-fix 2026-11-06). +const METRIC_RE = /^METRIC\s+([\w.µ-]+)=(\S+)$/; const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]); @@ -14,20 +17,38 @@ const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]); export function parseMetricLines( output: unknown, metricName?: string, -): { metrics: Record; primary: number | undefined } { +): { + metrics: Record; + primary: number | undefined; + nearMisses: string[]; +} { const metrics: Record = {}; let primary: number | undefined; + // Near-miss diagnostics: a line that looks like a METRIC row but fails the + // parse (bad name charset, non-numeric value) must not vanish silently — a + // silently dropped primary metric only surfaces later as a confusing + // "metric must be a finite number" on keep. Same never-silent principle as + // the ledger's corrupt-line diagnostic. + const nearMisses: string[] = []; for (const line of String(output ?? "").split("\n")) { - const m = METRIC_RE.exec(line.trim()); - if (!m) continue; - const [, name, rawValue] = m; - if (FORBIDDEN_KEYS.has(name)) continue; - const value = Number(rawValue); - if (!Number.isFinite(value)) continue; - metrics[name] = value; - if (name === metricName) primary = value; + const trimmed = line.trim(); + if (!/^METRIC\s/.test(trimmed)) continue; + // Every METRIC-looking line must land in metrics or in nearMisses — the + // silent middle (regex passes, Number() fails; name charset rejects) is + // where primary metrics used to vanish (never-silent principle, same as + // the ledger's corrupt-line diagnostic). + const m = METRIC_RE.exec(trimmed); + const name = m?.[1]; + const value = + name != null && !FORBIDDEN_KEYS.has(name) ? Number(m?.[2]) : NaN; + if (name != null && Number.isFinite(value)) { + metrics[name] = value; + if (name === metricName) primary = value; + } else if (nearMisses.length < 10) { + nearMisses.push(trimmed); + } } - return { metrics, primary }; + return { metrics, primary, nearMisses }; } /** @@ -43,10 +64,7 @@ export function isBetter( return direction === "higher" ? current > best : current < best; } -/** - * MAD-based noise floor for the current segment. - * Returns null when there are fewer than 3 data points or MAD is 0. - */ +/** Median of the values (null for an empty list); the noise floor lives in computeConfidence. */ export function median(values: number[]): number | null { if (values.length === 0) return null; const sorted = [...values].sort((a, b) => a - b); @@ -60,12 +78,16 @@ export function computeConfidence({ values, baseline, best, + deterministic = false, }: { values: number[]; baseline: number | null; best: number | null; + /** Bitwise-reproducible benchmark: the noise floor is a single point, so no MAD ratio exists — any delta is exact signal, not a directional hint. */ + deterministic?: boolean; }): Confidence | null { if (values.length < 3 || baseline == null || best == null) return null; + if (deterministic) return null; const med = median(values); if (med == null || med === 0) return null; const deviations = values.map((v) => Math.abs(v - med)); @@ -81,13 +103,21 @@ export function computeConfidence({ /** * Enforce that a run_experiment command is (a wrapper around) the benchmark * script. Strips leading `FOO=bar` assignments and `env/time/nice/nohup` - * wrappers, then requires the core command to start with the measure script - * path. Returns the unwrapped command string, or null when the command is - * not the benchmark script. Prevents `evil; ./measure.sh` chained injection. + * wrappers (assignment values pass the same character whitelist as script + * args — a prefix is not a smuggling channel for `$(…)`/backticks), then + * requires the core command to start with the measure script path. Returns + * the unwrapped command string, or null when the command is not the + * benchmark script. Prevents `evil; ./measure.sh` chained injection. */ const WRAP_RE = /^(env|time|nice|nohup)\s+/; -// leading env assignment including its value: `FOO=1 ` or `FOO="a b" ` -const ASSIGN_RE = /^[A-Za-z_][A-Za-z0-9_]*=\S*\s*/; +// leading env assignment including its value: `FOO=1 ` or `FOO=a=b ` +const ASSIGN_RE = /^([A-Za-z_][A-Za-z0-9_]*=)(\S*)\s*/; +// The assignment VALUE goes through the same whitelist idea as the args below: +// bash executes a prefix assignment before the script runs, so `X=$(id)` or +// backticks would smuggle arbitrary commands past the lock. Values are +// restricted to the arg character class minus separators (`\S*` already +// excludes whitespace). +const ASSIGN_VALUE_RE = /^[\w./:+=-]*$/; export function unwrapMeasureCommand( command: unknown, @@ -100,7 +130,11 @@ export function unwrapMeasureCommand( let prev: string; do { prev = cmd; - cmd = cmd.replace(ASSIGN_RE, "").trim(); + const assign = ASSIGN_RE.exec(cmd); + if (assign) { + if (!ASSIGN_VALUE_RE.test(assign[2])) return null; + cmd = cmd.slice(assign[0].length).trim(); + } cmd = cmd.replace(WRAP_RE, "").trim(); } while (cmd !== prev); if (!cmd) return null; @@ -154,12 +188,28 @@ export function isStopReached( * Normalize a hypothesis/description for comparison: lowercase, strip * non-alphanumerics, sort tokens. Returns null when there is no signal. */ +const CJK_RE = /[\u4e00-\u9fff]/; + +/** + * A long CJK token is a compound ("试埃氏筛变体"), so treating it as one + * whitespace-delimited word makes every variant look unrelated. Bigrams keep + * the shared prefix/suffix signal without a segmentation dependency; English + * tokens keep their existing word-level behavior. + */ +function hypothesisTokens(token: string): string[] { + if (!CJK_RE.test(token) || token.length <= 2) return [token]; + const out: string[] = []; + for (let i = 0; i < token.length - 1; i++) out.push(token.slice(i, i + 2)); + return out; +} + export function normalizeHypothesis(text: unknown): string | null { const tokens = String(text ?? "") .toLowerCase() .replace(/[^a-z0-9\u4e00-\u9fff]+/g, " ") .split(/\s+/) - .filter((t) => t.length >= 2); + .filter((t) => t.length >= 2) + .flatMap(hypothesisTokens); if (tokens.length === 0) return null; return [...tokens].sort().join(" "); } @@ -181,13 +231,17 @@ export function hypothesesSimilar( /** * Direction label for a run: prefer asi.hypothesis, else description; take the - * leading clause (up to first comma/period/semicolon), capped at 40 chars. + * leading clause (up to first comma/semicolon), capped at 40 chars. The half + * width period is deliberately NOT a clause boundary: decimals ("lambda_l2 + * 1.0", "bagging_fraction 0.8") would otherwise be cut at the dot and the + * memory injection would present "lambda_l2 1" / "bagging_fraction 0" as the + * tried direction (evidence 2026-11-05, ml-auto dota2-wave1). */ export function directionLabel(run: RunLike): string { const raw = run?.asi?.hypothesis || run?.description || ""; const clause = String(raw) - .split(/[,.;，。；]/)[0] + .split(/[,;，。；]/)[0] ?.trim() || ""; return clause.length > 40 ? clause.slice(0, 40) + "…" @@ -199,12 +253,17 @@ export function directionLabel(run: RunLike): string { * hypotheses. Returns { doomLoop, pattern } or null. * - repeat: last 3 runs have similar normalized hypotheses. * - oscillate: last 4 runs are [X, Y, X, Y] (X~X, Y~Y, X!~Y). + * Rows annotated `asi.infra: true` (infrastructure failure: network outage, + * missing dependency, platform throttling) are dropped before the window is + * taken — a hypothesis interrupted repeatedly by a broken environment is not + * the agent thrashing between ideas. */ export function detectDoomLoop( runs: RunLike[], { window = 6 } = {}, ): { doomLoop: boolean; pattern: "repeat" | "oscillate" } | null { const norm = runs + .filter((r) => r?.asi?.infra !== true) .filter((r) => r.description || r?.asi?.hypothesis) .slice(-window) .map((r) => normalizeHypothesis(r.asi?.hypothesis || r.description)) @@ -254,7 +313,15 @@ export function detectPlateau( } = {}, ): boolean { const valid = runs - .filter((r) => r.metric != null && Number.isFinite(r.metric)) + .filter( + (r) => + // Crash rows measure nothing (ADR-10; legacy rows carry a 0 + // placeholder) — they are not data points for the plateau window. + // Noop rows DO count: a noop re-measures the metric on unchanged + // code, so it is a legitimate trend data point (the repeat:3 + // confirmation flow relies on it keeping the window flat). + r.status !== "crash" && r.metric != null && Number.isFinite(r.metric), + ) .slice(-window); if (valid.length < window) return false; const first = valid[0].metric; diff --git a/external_plugins/autoresearch/mcp/lib/frozen.ts b/external_plugins/autoresearch/mcp/lib/frozen.ts new file mode 100644 index 00000000..9ba91592 --- /dev/null +++ b/external_plugins/autoresearch/mcp/lib/frozen.ts @@ -0,0 +1,40 @@ +// Frozen session files and the path key the write guard compares. +// +// Keys are POSIX-style because that is how .auto/ paths are written +// everywhere else. path.relative() yields backslashes on Windows; Windows and +// macOS default to case-insensitive filesystems, so comparing the raw relative +// path against these keys would silently allow rewrites there. + +export const FROZEN_FILES: ReadonlySet = new Set([ + ".auto/measure.sh", + ".auto/checks.sh", + ".auto/goal.md", + ".auto/scope.md", +]); + +export const FROZEN_REASONS: Record = { + ".auto/measure.sh": + "the benchmark metric must not change during the loop. If you really need a new metric, the recorded file must go first: have the user delete it (or delete it with an explicitly user-approved command) — this guard only protects a file that exists — then set up the new benchmark and start a new segment with init_experiment.", + ".auto/checks.sh": "the correctness gate must not change during the loop.", + ".auto/goal.md": + "the deep goal is the root the loop serves; it was recorded verbatim from the user and cannot be rewritten. Surface-goal wording is revisable via the harness-edit step's editable surfaces, or a new segment via init_experiment.", + ".auto/scope.md": + "the scope boundary was agreed with the user and cannot be changed by the loop — widening or narrowing it is the user's call: ask them to edit this file, or init_experiment for a new segment.", +}; + +/** + * Normalize a relative path to the key form used by FROZEN_FILES. + * Windows: forward slashes + lowercase (the filesystem is case-insensitive). + * POSIX: verbatim, so case-sensitive names stay distinct. + */ +export function frozenKey( + rel: string, + platform: NodeJS.Platform = process.platform, +): string { + // Windows and macOS default to case-insensitive filesystems. Treating a + // case variant as a different path would let .auto/MEASURE.SH slip past + // the write guard while opening the same file; lowercasing is the safe + // direction (a rare case-sensitive macOS volume only gets a false deny). + if (platform !== "win32" && platform !== "darwin") return rel; + return rel.replace(/\\/g, "/").toLowerCase(); +} diff --git a/external_plugins/autoresearch/mcp/lib/git.ts b/external_plugins/autoresearch/mcp/lib/git.ts index d161b85b..376853a8 100644 --- a/external_plugins/autoresearch/mcp/lib/git.ts +++ b/external_plugins/autoresearch/mcp/lib/git.ts @@ -1,32 +1,141 @@ // Git operations for the experiment loop (ADR-2 semantics): // keep → commit with `experiment:` prefix + structured Result JSON. -// discard/crash/checks_failed → drop working-tree changes, exempt `.auto/`. +// discard/crash/checks_failed → drop working-tree changes, exempt session state +// (`.auto/`, the host session dir `.zcode/` and the generated dashboard). import { execFileSync } from "node:child_process"; +import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { DASHBOARD_FILE } from "./ledger.ts"; + +// Session state dirs exempt from every git consumption site (keep staging, +// dirty check, rollback restore/clean). Single source of truth: the sites +// speak different pathspec dialects, and per-site enumeration missed `.zcode` +// the same way `.auto` once did (evidence 2026-11-05: rollback clean -fd +// deleted the host's .zcode/plans on every discard, and dirtyFiles counted +// session files as tree changes, deadlocking the crash gate). +const SESSION_STATE_DIRS = [".auto", ".zcode"]; + +// Standard install locations a slim GUI PATH can hide. On macOS the +// launchd-default PATH still contains /usr/bin, whose git (the CLT shim) +// covers most GUI launches — but homebrew-only machines without the CLT are +// left with git at /opt/homebrew/bin, invisible to a GUI-launched server. +// On Windows a GUI PATH can miss Git for Windows entirely; its installer +// default is the cmd\git.exe shim under Program Files. +const GIT_CANDIDATES = + process.platform === "win32" + ? [ + "C:\\Program Files\\Git\\cmd\\git.exe", + "C:\\Program Files (x86)\\Git\\cmd\\git.exe", + ] + : [ + "/opt/homebrew/bin/git", + "/usr/local/bin/git", + "/usr/bin/git", + "/bin/git", + ]; + +function probeGit(path: string): boolean { + try { + execFileSync(path, ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +let cachedGit: string | null = null; + +/** + * Resolve the git binary: PATH first (one probe, then cached), falling back + * to the standard install locations. Injected candidates (tests) bypass the + * PATH probe and the cache. Throws with a loud message when nothing + * resolves — every spawn in this module goes through here, so a missing git + * surfaces as one clear error instead of per-call ENOENT. + */ +export function resolveGit(candidates?: string[]): string { + if (candidates === undefined) { + if (cachedGit) return cachedGit; + if (probeGit("git")) { + cachedGit = "git"; + return "git"; + } + for (const path of GIT_CANDIDATES) { + if (existsSync(path) && probeGit(path)) { + cachedGit = path; + return path; + } + } + } else { + for (const path of candidates) { + if (probeGit(path)) return path; + } + } + throw new Error( + "no usable git on PATH or standard install locations; install git or make sure it is on the server's PATH", + ); +} export function git(cwd: string, args: string[]): string { - return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); + return execFileSync(resolveGit(), args, { cwd, encoding: "utf8" }).trim(); } export function isGitRepo(cwd: string): boolean { + // resolveGit() throws when no git exists at all — that is a different + // failure from "this directory is not a repository" and callers must be + // able to tell them apart. Only the rev-parse failure maps to false. + const gitBin = resolveGit(); try { - execFileSync("git", ["rev-parse", "--git-dir"], { cwd, stdio: "ignore" }); + execFileSync(gitBin, ["rev-parse", "--git-dir"], { + cwd, + stdio: "ignore", + }); return true; } catch { return false; } } +/** Whether the repository has at least one commit (HEAD resolves). */ +export function hasCommit(cwd: string): boolean { + try { + execFileSync(resolveGit(), ["rev-parse", "--verify", "HEAD"], { + cwd, + stdio: "ignore", + }); + return true; + } catch { + return false; + } +} + +/** + * The real changes behind isDirty, named: one porcelain line per path + * (session state excluded, untracked included). Unlike isDirty this lets a + * status failure throw — a post-rollback verification must never mistake an + * unreadable status for a clean tree. + */ +export function dirtyFiles(cwd: string): string[] { + return git(cwd, [ + "status", + "--porcelain", + "--", + ".", + ...SESSION_STATE_DIRS.map((d) => `:(exclude)${d}`), + `:(exclude)${DASHBOARD_FILE}`, + ]) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); +} + /** - * Whether the working tree has real (non-`.auto/`) changes. Session-file - * writes (ledger, config) must not count: e.g. the crash-unresolved gate + * Whether the working tree has real changes (session state excluded). Ledger, + * config and dashboard writes must not count: e.g. the crash-unresolved gate * would otherwise block forever right after logging the crash itself. */ export function isDirty(cwd: string): boolean { try { - return ( - git(cwd, ["status", "--porcelain", "--", ".", ":(exclude).auto"]).length > - 0 - ); + return dirtyFiles(cwd).length > 0; } catch { return false; } @@ -36,21 +145,85 @@ export function shortHash(cwd: string): string { return git(cwd, ["rev-parse", "--short=7", "HEAD"]); } +/** Short HEAD hash for the dashboard footer pin; null outside a repo / on failure. */ +export function safeShortHash(cwd: string): string | null { + try { + return isGitRepo(cwd) ? shortHash(cwd) : null; + } catch { + return null; + } +} + +/** Full HEAD sha, or null on an unborn branch / git failure. */ +export function headSha(cwd: string): string | null { + if (!hasCommit(cwd)) return null; + try { + return git(cwd, ["rev-parse", "HEAD"]).trim(); + } catch { + return null; + } +} + +/** + * Undo the experiment keep commit: move the branch back to `sha` and reset + * the index to it, leaving the working tree untouched — the committed changes + * return to the tree as unstaged modifications, i.e. exactly the pre-keep + * state. Used when the ledger append fails after a successful commit: a + * commit without its ledger row would silently diverge from the audit chain + * (after the undo, the retry looks exactly like the first attempt). + */ +export function undoCommitTo(cwd: string, sha: string): void { + git(cwd, ["reset", "-q", "--mixed", sha]); +} + /** - * Commit all tracked+untracked changes as one experiment, excluding the - * `.auto/` session dir (ledger noise must not ride along, and a keep with - * only session-file changes must hit the "nothing to commit" path). + * Commit all tracked+untracked changes as one experiment, excluding session + * state (`.auto/`, `.zcode/` and the generated dashboard): a keep with only + * session-file changes must hit the "nothing to commit" path. * Returns the short hash, or null when there is nothing to commit. */ export function commitExperiment( cwd: string, { description, result }: { description: string; result: unknown }, ): string | null { - git(cwd, ["add", "-A", "--", ".", ":(exclude).auto"]); + // Unstage the whole index first (working tree untouched): the commit below + // must describe this experiment and nothing else. Excluding paths from add + // does not remove files already in the index, so pre-staged state from + // outside the research dir would otherwise ride into the commit — and, when + // the research dir itself has no changes, would fake a non-empty experiment + // (a keep whose commit contains none of the experiment). + git(cwd, ["reset", "-q", "HEAD"]); + // Stage with `-- .`, then unstage session paths again. Two constraints meet + // here: + // - No exclude pathspec: an exclude that covers a gitignored path makes add + // exit 1 ("paths are ignored"), which aborted every keep in repos with + // gitignored session state (evidence 2026-11-01: ml-auto; reproduced on + // git 2.55.0 with the exact quoted argv). Plain `-- .` skips ignored + // files instead of failing on them. + // - Keep the pathspec: it scopes the commit to the research dir. A bare + // `git add -A` is repo-wide, so under `workingDir: work/` it also staged + // the project's own `.auto/` (the post-add reset is cwd-relative and + // cannot reach it) and unrelated files elsewhere in the repo. + // The post-add reset removes session paths even when they are tracked or + // pre-staged; with the index cleared above, the cached diff below answers + // exactly "did this experiment change anything". `.zcode` is the host's + // session-state dir (plans, session files) and often NOT gitignored — left + // in, `add -A -- .` sweeps it into the keep commit (evidence 2026-11-05, + // ml-auto: .zcode/plans/plan-sess_*.md landed in an experiment commit). + // Non-existent pathspecs here are a silent no-op (git reset exit 0). + git(cwd, ["add", "-A", "--", "."]); + git(cwd, [ + "reset", + "-q", + "HEAD", + "--", + ...SESSION_STATE_DIRS, + DASHBOARD_FILE, + ]); // git diff --cached --quiet exits 0 when there are no staged changes. const hasStaged = (() => { try { - execFileSync("git", ["diff", "--cached", "--quiet"], { + execFileSync(resolveGit(), ["diff", "--cached", "--quiet"], { cwd, stdio: "ignore", }); @@ -61,37 +234,144 @@ export function commitExperiment( })(); if (!hasStaged) return null; const body = `experiment: ${description}\n\nResult: ${JSON.stringify(result)}`; - execFileSync("git", ["commit", "-m", body], { cwd, stdio: "ignore" }); + // --no-verify isolates the experiment commit from the target repo's own git + // hooks: a husky/lint-staged style pre-commit would rewrite staged content + // (breaking "the commit describes exactly this experiment") or fail the + // commit for reasons unrelated to the research. stderr is captured so a + // failing commit surfaces git's actual diagnostics instead of a bare + // "Command failed" (evidence 2026-11-04: hook-modified content committed, + // and hook-failure error with zero diagnostics, both reproduced). + try { + execFileSync(resolveGit(), ["commit", "--no-verify", "-m", body], { + cwd, + stdio: ["ignore", "ignore", "pipe"], + encoding: "utf8", + }); + } catch (err) { + const stderr = + err && typeof err === "object" && "stderr" in err + ? String((err as { stderr?: unknown }).stderr).trim() + : ""; + throw new Error( + `git commit failed: ${stderr || (err instanceof Error ? err.message : String(err))}. The experiment changes remain staged in the index (pre-staged entries elsewhere were unstaged while attempting the keep).`, + ); + } return shortHash(cwd); } /** - * Discard every working-tree + index change while keeping the `.auto/` - * session directory intact. Uses `checkout HEAD` (not `checkout --`) so - * staged-but-uncommitted experiment changes are reverted to HEAD too. + * Declare `.zcode` in the repo's local exclude file (`.git/info/exclude`) so + * untracked host session files are invisible to `add -A`, `status` and + * `clean -fd` alike — git-native ignore instead of per-site enumeration. + * Per-session-file evidence (2026-11-05, ml-auto): the enumeration layer alone + * shipped three gaps in one day (keep staging swept .zcode/plans into a + * commit; clean -fd deleted the host session dir on every discard; dirtyFiles + * deadlocked the crash gate). Skipped when the path is already ignored + * (project `.gitignore` covers it); idempotent when the declaration is already + * present; any failure is swallowed — the SESSION_STATE_DIRS enumeration is + * the correctness backstop, this layer is recurrence prevention. + * Returns whether the declaration was newly written. */ -export function rollbackWorkingTree(cwd: string): void { - execFileSync( - "git", - ["checkout", "HEAD", "--", ".", ":(exclude,glob)**/.auto/**"], - { cwd, stdio: "ignore" }, - ); +export function ensureSessionStateIgnored(cwd: string): boolean { + try { + execFileSync(resolveGit(), ["check-ignore", "-q", ".zcode"], { + cwd, + stdio: "ignore", + }); + return false; // exit 0: already ignored, nothing to declare + } catch { + // exit 1 = not ignored (the expected path); any other failure falls + // through and attempts the write anyway — the append is harmless. + } + try { + const gitDir = git(cwd, ["rev-parse", "--git-dir"]); + const excludeFile = resolve(cwd, gitDir, "info", "exclude"); + let current = ""; + try { + current = readFileSync(excludeFile, "utf8"); + } catch { + // missing or unreadable → created by the append below (info/ may not + // exist on bare or sparsely provisioned repos) + } + if (current.split(/\r?\n/).includes(".zcode")) return false; + mkdirSync(dirname(excludeFile), { recursive: true }); + appendFileSync(excludeFile, ".zcode\n"); + return true; + } catch { + return false; + } +} + +/** + * Discard every working-tree + index change while keeping session state + * (`.auto/`, `.zcode/` and the generated dashboard) intact. Uses `checkout + * HEAD` (not `checkout --`) so staged-but-uncommitted experiment changes are + * reverted too. + * + * Returns what the rollback did: `rolledBack` = the working-tree files the + * restore disposed of (pre-restore dirtyFiles — tracked reverts and untracked + * removes alike), `restoredTo` = the HEAD short hash the tree was restored to, + * `residual` = the post-rollback dirtyFiles so the caller can surface them + * instead of assuming the restore happened. Evidence 2026-10-29 (nano-jev + * V1 run 6): a discard once returned ok while the challenger stayed in the + * working tree — git exited 0 without restoring, intermittently (runs 2–5 + * rolled back fine; Windows file-lock or pathspec race suspected, root cause + * unconfirmed). stdio-ignored with no post-check made that silent, and every + * later read would sit on un-rolled-back inputs. So: restore with stderr + * captured (retry once — transient locks clear between attempts), then + * verify. Verification is root-cause-agnostic: any way the restore can + * silently fail ends up as a loud, named residual. + */ +export function rollbackWorkingTree(cwd: string): { + rolledBack: string[]; + restoredTo: string | null; + residual: string[]; +} { + const rolledBack = dirtyFiles(cwd); + const restore = () => + execFileSync( + resolveGit(), + [ + "checkout", + "HEAD", + "--", + ".", + ...SESSION_STATE_DIRS.map((d) => `:(exclude,glob)**/${d}/**`), + `:(exclude)${DASHBOARD_FILE}`, + ], + { cwd, encoding: "utf8" }, + ); + try { + restore(); + } catch { + try { + restore(); + } catch (retryError) { + const stderr = + retryError && typeof retryError === "object" && "stderr" in retryError + ? String((retryError as { stderr?: unknown }).stderr) + : String(retryError); + throw new Error(`rollback restore failed twice: ${stderr.trim()}`); + } + } // Unstage anything (e.g. accidentally staged .auto content) without touching files. - execFileSync("git", ["reset", "-q"], { cwd, stdio: "ignore" }); + execFileSync(resolveGit(), ["reset", "-q"], { cwd, stdio: "ignore" }); execFileSync( - "git", + resolveGit(), [ "clean", "-fd", + ...SESSION_STATE_DIRS.flatMap((d) => ["-e", d, "-e", `${d}/`]), "-e", - ".auto", - "-e", - ".auto/", - "-e", - "autoresearch-dashboard.html", + DASHBOARD_FILE, ], { cwd, stdio: "ignore" }, ); + return { + rolledBack, + restoredTo: safeShortHash(cwd), + residual: dirtyFiles(cwd), + }; } export function currentBranch(cwd: string): string { diff --git a/external_plugins/autoresearch/mcp/lib/harness.ts b/external_plugins/autoresearch/mcp/lib/harness.ts new file mode 100644 index 00000000..6627b43f --- /dev/null +++ b/external_plugins/autoresearch/mcp/lib/harness.ts @@ -0,0 +1,47 @@ +// Editable harness surfaces and the mandatory ledger trace for them. +// +// The harness-edit step's discipline (edit rows, verdicts, experience.md) is +// advisory; the FACT of a surface change is not. The server hashes the +// editable files at the accounting checkpoints (run_experiment / +// log_experiment / export_dashboard) and appends {type:"harness_edit"} rows +// for anything that moved — including changes made by hand, which no tool +// hook could see. config.json is deliberately excluded: server-managed keys +// (pendingChecksFailed, benchmarkHashes, harnessHashes itself) rewrite it +// constantly and would self-trigger noise rows. + +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +/** `.auto/`-relative keys, same key style as frozen.ts. */ +export const EDITABLE_SURFACE_FILES: readonly string[] = [ + ".auto/prompt.md", + ".auto/ideas.md", + ".auto/experience.md", + ".auto/hooks/before.sh", + ".auto/hooks/after.sh", +]; + +export function sha256File(file: string): string | null { + if (!existsSync(file)) return null; + return createHash("sha256").update(readFileSync(file)).digest("hex"); +} + +/** Current digests for the editable surface, keyed by `.auto/`-relative path. */ +export function currentHarnessHashes( + cwd: string, +): Record { + const hashes: Record = {}; + for (const file of EDITABLE_SURFACE_FILES) { + hashes[file] = sha256File(join(cwd, file)); + } + return hashes; +} + +/** Files whose digest moved from baseline to current. Pure, for tests. */ +export function diffHarnessEdits( + baseline: Record, + current: Record, +): string[] { + return Object.keys(current).filter((f) => baseline[f] !== current[f]); +} diff --git a/external_plugins/autoresearch/mcp/lib/ledger.ts b/external_plugins/autoresearch/mcp/lib/ledger.ts index 3666ad8f..d85e9f59 100644 --- a/external_plugins/autoresearch/mcp/lib/ledger.ts +++ b/external_plugins/autoresearch/mcp/lib/ledger.ts @@ -5,23 +5,29 @@ import { appendFileSync, existsSync, mkdirSync, + openSync, + readSync, + closeSync, + statSync, } from "node:fs"; import { join } from "node:path"; import { computeConfidence, isBetter, detectPlateau } from "./experiment.ts"; import type { + Confidence, + LedgerConfig, LedgerEntry, LedgerRun, SessionConfig, SessionState, } from "./types.ts"; -export const AUTO_DIR = ".auto"; +const AUTO_DIR = ".auto"; export const LOG_FILE = join(AUTO_DIR, "log.jsonl"); -export const PROMPT_FILE = join(AUTO_DIR, "prompt.md"); -export const MEASURE_FILE = join(AUTO_DIR, "measure.sh"); -export const CHECKS_FILE = join(AUTO_DIR, "checks.sh"); -export const CONFIG_FILE = join(AUTO_DIR, "config.json"); -export const IDEAS_FILE = join(AUTO_DIR, "ideas.md"); +const PROMPT_FILE = join(AUTO_DIR, "prompt.md"); +const MEASURE_FILE = join(AUTO_DIR, "measure.sh"); +const CHECKS_FILE = join(AUTO_DIR, "checks.sh"); +const CONFIG_FILE = join(AUTO_DIR, "config.json"); +const IDEAS_FILE = join(AUTO_DIR, "ideas.md"); export const DASHBOARD_FILE = "autoresearch-dashboard.html"; export function autoPaths(cwd: string): { @@ -46,29 +52,58 @@ export function autoPaths(cwd: string): { }; } -export function ensureAutoDir(cwd: string): void { +function ensureAutoDir(cwd: string): void { mkdirSync(join(cwd, AUTO_DIR), { recursive: true }); } +/** Whether the file's last byte is a newline (empty file counts as yes). */ +function endsWithNewline(path: string): boolean { + const size = statSync(path).size; + if (size === 0) return true; + const fd = openSync(path, "r"); + try { + const b = Buffer.alloc(1); + readSync(fd, b, 0, 1, size - 1); + return b[0] === 0x0a; + } finally { + closeSync(fd); + } +} + export function appendLedgerEntry(cwd: string, entry: LedgerEntry): void { ensureAutoDir(cwd); - appendFileSync(join(cwd, LOG_FILE), JSON.stringify(entry) + "\n", "utf8"); + const log = join(cwd, LOG_FILE); + // Line-boundary self-heal: a previous write killed mid-line (SIGKILL, + // ENOSPC) leaves a truncated record without a trailing newline, and a blind + // append would splice the new entry onto it — corrupting BOTH records. The + // repair only adds the missing newline; the truncated line stays as-is + // (append-only ledger, no in-place rewrite) and surfaces via readLedger's + // corrupt-line diagnostic. + if (existsSync(log) && !endsWithNewline(log)) + appendFileSync(log, "\n", "utf8"); + appendFileSync(log, JSON.stringify(entry) + "\n", "utf8"); } export function readLedger(cwd: string): LedgerEntry[] { const log = join(cwd, LOG_FILE); if (!existsSync(log)) return []; - return readFileSync(log, "utf8") - .split("\n") - .filter((l) => l.trim()) - .map((l) => { - try { - return JSON.parse(l) as LedgerEntry; - } catch { - return null; - } - }) - .filter((e): e is LedgerEntry => e != null); + const out: LedgerEntry[] = []; + const lines = readFileSync(log, "utf8").split("\n"); + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + if (!l.trim()) continue; + try { + out.push(JSON.parse(l) as LedgerEntry); + } catch { + // Never silent: a dropped run row shifts every later run number and the + // audit rejects all writes with an error that cannot point here. stderr + // is the diagnostic channel (stdout belongs to JSON-RPC / hook output). + process.stderr.write( + `[autoresearch] ${LOG_FILE} line ${i + 1} is corrupt and was skipped: ${l.slice(0, 120)}\n`, + ); + } + } + return out; } /** @@ -85,6 +120,7 @@ export function rebuildState( consecutiveFailures?: number; plateauWindow?: number; plateauMinImprovement?: number; + deterministic?: boolean; } = {}, ): SessionState { const entries = readLedger(cwd); @@ -94,9 +130,8 @@ export function rebuildState( runs: [], baseline: null, best: null, - lastRunChecksFailed: false, lastRun: null, - totalExperiments: 0, + lastRunEver: null, consecutiveFailures: 0, confidence: null, plateau: false, @@ -109,6 +144,12 @@ export function rebuildState( state.runs = []; state.baseline = null; state.best = null; + // A config row is a segment boundary: failure streaks and the previous + // run's hook payload must not leak into the new segment. The checks + // verdict lives in config's pendingChecksFailed (cleared wholesale at + // segment boundaries), not as a derived field here. + state.consecutiveFailures = 0; + state.lastRun = null; } else if (e.type === "run") { const run: LedgerRun = { ...e, @@ -116,8 +157,15 @@ export function rebuildState( config: state.config, }; state.runs.push(run); - state.totalExperiments += 1; - if (state.baseline == null && run.metric != null) + // Crash rows measure nothing (ADR-10: metric null; pre-ADR-10 rows carry + // a 0 placeholder) — they must not become the baseline, or a legacy + // ledger shows baseline 0 and absurd deltas. Matches validateLedger, + // which already skips crash rows here. + if ( + state.baseline == null && + run.metric != null && + run.status !== "crash" + ) state.baseline = run.metric; if (run.status === "keep" && run.metric != null) { const dir = state.config?.direction ?? "lower"; @@ -129,16 +177,16 @@ export function rebuildState( if (run.status === "keep" || run.status === "noop") state.consecutiveFailures = 0; else state.consecutiveFailures += 1; - // Per-run overwrite (no one-way latch): the flag always describes the - // latest ledger run. Status alone marks checks failure; the explicit - // field covers hand-written/legacy rows. - state.lastRunChecksFailed = - run.checksFailed === true || run.status === "checks_failed"; state.lastRun = run; + // Not reset at segment boundaries: the crash+dirty-tree audit gate reads + // this, and a dirty tree is physical state init_experiment can't clear. + state.lastRunEver = run; } } - // Confidence over the current segment's values. + // Confidence over the current segment's values. Crash rows (including + // legacy 0-placeholder ones) measure nothing and stay out of the value set. const values = state.runs + .filter((r) => r.status !== "crash") .map((r) => r.metric) .filter((v): v is number => v != null && Number.isFinite(v)); if (state.config && values.length > 0) { @@ -146,6 +194,7 @@ export function rebuildState( values, baseline: state.baseline, best: state.best, + deterministic: options.deterministic === true, }); } else { state.confidence = null; @@ -166,18 +215,88 @@ export function rebuildState( return state; } +// --------------------------------------------------------------------------- +// Full-session history (archive dashboard only; loop control stays on +// rebuildState's segment-scoped view) +// --------------------------------------------------------------------------- + /** - * Delta of a run's metric against the segment baseline, direction-aware: - * positive = improvement (lower metric + baseline was higher, or higher metric). + * Flat whole-ledger view: the session reads as one run sequence. A config row + * never splits the display — it only refreshes the title/metric/direction + *口径 (the latest config wins). */ -export function deltaFor( - state: SessionState, - metric: number | null | undefined, -): number | null { - if (metric == null || state.baseline == null) return null; - const dir = state.config?.direction ?? "lower"; - const raw = metric - state.baseline; - return dir === "higher" ? raw : -raw; +export interface SessionHistory { + config: LedgerConfig | null; + runs: LedgerRun[]; + totalRuns: number; + kept: number; + firstMetric: number | null; + bestMetric: number | null; + lastRunAt: string | null; + confidence: Confidence | null; +} + +/** + * Pure view over the whole ledger, flattened. Runs keep their ledger order + * regardless of config rows; the latest config row supplies the display + *口径 (name, metric, direction, unit). A run before any config row is + * rendered with the neutral fallbacks (hand-written ledgers). + */ +export function rebuildHistory(entries: LedgerEntry[]): SessionHistory { + const runs: LedgerRun[] = []; + let config: LedgerConfig | null = null; + let firstMetric: number | null = null; + let lastRunAt: string | null = null; + for (const e of entries) { + if (e.type === "config") { + config = e; + } else if (e.type === "run") { + runs.push(e); + // Crash rows measure nothing (see rebuildState): a legacy 0-placeholder + // must not become the dashboard's global baseline. + if ( + firstMetric == null && + e.metric != null && + Number.isFinite(e.metric) && + e.status !== "crash" + ) + firstMetric = e.metric; + lastRunAt = e.timestamp ?? null; + } + } + // Best over keep runs, direction-aware by the latest config. + const dir = config?.direction ?? "lower"; + let bestMetric: number | null = null; + for (const r of runs) { + if (r.status !== "keep" || r.metric == null || !Number.isFinite(r.metric)) + continue; + if (bestMetric == null || isBetter(r.metric, bestMetric, dir)) + bestMetric = r.metric; + } + // Confidence over all measured values against the global baseline/best. + // Crash rows (legacy 0 placeholders included) measure nothing. + const values = runs + .filter((r) => r.status !== "crash") + .map((r) => r.metric) + .filter((v): v is number => v != null && Number.isFinite(v)); + const confidence = + config && values.length > 0 + ? computeConfidence({ + values, + baseline: firstMetric, + best: bestMetric, + }) + : null; + return { + config, + runs, + totalRuns: runs.length, + kept: runs.filter((r) => r.status === "keep").length, + firstMetric, + bestMetric, + lastRunAt, + confidence, + }; } export function readSessionConfig(cwd: string): SessionConfig { @@ -190,6 +309,27 @@ export function readSessionConfig(cwd: string): SessionConfig { } } +/** + * Plateau calibration from `.auto/config.json`, sanitized for rebuildState: + * invalid values fall back to the defaults instead of erroring (calibration + * is a convenience, not a gate). Shared by the server and the Stop hook so + * detection and hint wording stay on one口径. + */ +export function plateauCalibration(cwd: string): { + minImprovement?: number; + window?: number; + deterministic: boolean; +} { + const cfg = readSessionConfig(cwd); + const minRaw = Number(cfg.plateauMinImprovement); + const winRaw = Number(cfg.plateauWindow); + return { + minImprovement: Number.isFinite(minRaw) && minRaw >= 0 ? minRaw : undefined, + window: Number.isInteger(winRaw) && winRaw >= 2 ? winRaw : undefined, + deterministic: cfg.deterministic === true, + }; +} + export function writeDashboard(cwd: string, html: string): string { writeFileSync(join(cwd, DASHBOARD_FILE), html, "utf8"); return join(cwd, DASHBOARD_FILE); diff --git a/external_plugins/autoresearch/mcp/lib/paths.ts b/external_plugins/autoresearch/mcp/lib/paths.ts index adaf36cb..6a840010 100644 --- a/external_plugins/autoresearch/mcp/lib/paths.ts +++ b/external_plugins/autoresearch/mcp/lib/paths.ts @@ -1,21 +1,53 @@ // Resolve the effective research directory. `.auto/config.json` in the project // dir may set `workingDir` (relative to the project or absolute); when it // exists, all experiment operations happen there (config stays in the project). +// +// Three outcomes, deliberately different: +// - no workingDir configured → silent fallback to the project dir (normal); +// - config.json unparseable → stderr diagnostic, then fallback (a corrupt +// config must not brick hooks, but must not be invisible either); +// - workingDir set but the target is missing / not a directory → THROW. +// Silently falling back here would repoint the destructive paths +// (`rollbackWorkingTree`: checkout HEAD + clean -fd) at the project root and +// destroy uncommitted work outside the research dir. Failing loud is the +// only safe direction: better no experiment than a rollback in the wrong tree. +// Hooks wrap this call and fail open with a diagnostic (hook discipline); +// the MCP server lets it throw at startup so the misconfiguration surfaces +// as a clear error instead of misdirected git operations. import { readFileSync, statSync } from "node:fs"; import { resolve, isAbsolute, join } from "node:path"; export function resolveWorkCwd(projectCwd: string): string { + let raw: string; try { - const cfg = JSON.parse( - readFileSync(join(projectCwd, ".auto", "config.json"), "utf8"), - ) as { workingDir?: unknown }; - const wd = cfg.workingDir; - if (typeof wd === "string" && wd.trim()) { - const target = isAbsolute(wd) ? wd : resolve(projectCwd, wd); - if (statSync(target).isDirectory()) return target; - } + raw = readFileSync(join(projectCwd, ".auto", "config.json"), "utf8"); + } catch { + return projectCwd; // no config → project dir + } + let cfg: { workingDir?: unknown }; + try { + cfg = JSON.parse(raw) as { workingDir?: unknown }; } catch { - /* no config or no workingDir → project dir */ + process.stderr.write( + "[autoresearch] warning: .auto/config.json is not valid JSON; falling back to the project directory\n", + ); + return projectCwd; + } + const wd = cfg.workingDir; + if (typeof wd === "string" && wd.trim()) { + const target = isAbsolute(wd) ? wd : resolve(projectCwd, wd); + let isDir = false; + try { + isDir = statSync(target).isDirectory(); + } catch { + isDir = false; + } + if (!isDir) { + throw new Error( + `.auto/config.json workingDir "${wd}" (resolved: ${target}) is missing or not a directory; fix or remove workingDir before running experiments`, + ); + } + return target; } return projectCwd; } diff --git a/external_plugins/autoresearch/mcp/lib/spill.ts b/external_plugins/autoresearch/mcp/lib/spill.ts new file mode 100644 index 00000000..cd206042 --- /dev/null +++ b/external_plugins/autoresearch/mcp/lib/spill.ts @@ -0,0 +1,42 @@ +// Spill-file hygiene for run_experiment's output overflow (server.ts). +// A spilled benchmark log is read at most until the tool call returns its +// tail — afterwards it only lingers in the OS temp dir (which Windows in +// particular never cleans) holding disk space and whatever the benchmark +// printed. Sweeping stale files at spill time bounds the accumulation +// without touching the file the current call is about to write. +import { readdirSync, statSync, unlinkSync } from "node:fs"; +import { join } from "node:path"; + +export const SPILL_FILE_PREFIX = "autoresearch-experiment-"; + +/** + * Delete spilled experiment logs older than maxAgeMs. Best-effort: an + * unreadable temp dir or a file racing with another sweeper must never fail + * the benchmark run the sweep rides on. Returns how many files were removed. + */ +export function sweepStaleSpillFiles( + dir: string, + maxAgeMs: number, + now: number, +): number { + let names: string[]; + try { + names = readdirSync(dir); + } catch { + return 0; + } + let removed = 0; + for (const name of names) { + if (!name.startsWith(SPILL_FILE_PREFIX) || !name.endsWith(".log")) continue; + const file = join(dir, name); + try { + if (now - statSync(file).mtimeMs > maxAgeMs) { + unlinkSync(file); + removed += 1; + } + } catch { + /* gone or locked — leave it for the next sweep */ + } + } + return removed; +} diff --git a/external_plugins/autoresearch/mcp/lib/types.ts b/external_plugins/autoresearch/mcp/lib/types.ts index d39fac3b..d2882039 100644 --- a/external_plugins/autoresearch/mcp/lib/types.ts +++ b/external_plugins/autoresearch/mcp/lib/types.ts @@ -36,7 +36,10 @@ export interface LedgerRun { description?: string; commit?: string | null; checksFailed?: boolean; - failedGuard?: boolean; + /** Mechanical fact: the tree (session state excluded) at log time. */ + treeDirty?: boolean; + /** Working-tree residue a rollback could not restore (absent = clean). */ + residual?: string[]; /** confidence snapshot at record time (log_experiment); absent on legacy * rows and whenever the computation had too little data to judge. */ confidence?: ConfidenceSnapshot | null; @@ -56,7 +59,45 @@ export interface LedgerHook { timed_out: boolean; } -export type LedgerEntry = LedgerConfig | LedgerRun | LedgerHook; +/** `harness_edit` row written by the mandatory harness trace (server-side). */ +export interface LedgerHarnessEdit { + type: "harness_edit"; + ts: string; + segment: number; + file: string; + digest_before: string | null; + digest_after: string | null; +} + +/** `frozen_attempt` row written by guard-frozen on a denied frozen rewrite. */ +export interface LedgerFrozenAttempt { + type: "frozen_attempt"; + ts: string; + file: string; + tool: string | null; +} + +export type LedgerEntry = + | LedgerConfig + | LedgerRun + | LedgerHook + | LedgerHarnessEdit + | LedgerFrozenAttempt; + +/** + * Observation snapshot persisted by run_experiment (config.json + * `pendingObservation`) and consumed by log_experiment: run_experiment is the + * loop's only observer, and this snapshot is what makes every quantitative + * claim in the ledger a transcription of a measured fact instead of a free + * declaration. Overwritten by the next run; consumed by each successful log. + */ +export interface PendingObservation { + metric: number | null; + metrics: Record; + exit_code: number | null; + timed_out: boolean; + at: string; +} /** * Loose run shape consumed by analysis functions (isStopReached, @@ -92,9 +133,14 @@ export interface SessionState { runs: LedgerRun[]; baseline: number | null; best: number | null; - lastRunChecksFailed: boolean; + /** Segment-scoped: reset by config rows (before-hook payload, checks marker). */ lastRun: LedgerRun | null; - totalExperiments: number; + /** + * Ledger-wide last run row, NOT reset at segment boundaries: the + * crash+dirty-tree audit gate reads this, because an unrolled-back working + * tree is physical state that init_experiment must not launder. + */ + lastRunEver: LedgerRun | null; consecutiveFailures: number; confidence: Confidence | null; plateau: boolean; @@ -110,7 +156,28 @@ export interface SessionConfig { auditBypass?: boolean; autoresearchOff?: boolean; benchmarkHashes?: { measure: string | null; checks: string | null } | null; + /** Server-managed: editable-surface digests baseline for the harness trace. */ + harnessHashes?: Record | null; + /** + * `pending` prefix convention: keys named `pending…` are server cross-call + * mechanism state, cleared wholesale at segment boundaries (init_experiment + * / clear_experiments — resetPendingState). Never store user intent or a + * session baseline under this prefix; future keys are covered by the + * boundary reset automatically. + */ /** Server-managed: checks outcome of the latest run_experiment (keep gate). */ pendingChecksFailed?: boolean; + /** Server-managed: observation snapshot consumed by log_experiment's binding gates. */ + pendingObservation?: PendingObservation | null; + /** + * Loop calibration: plateau window in runs (integer ≥2, default 5) and the + * minimum relative improvement over that window (≥0, default 0.01). Set + * near the metric's real step size — the default 1% misfires on metrics + * that move in the 4th decimal (e.g. AUC). + */ + plateauWindow?: number; + plateauMinImprovement?: number; + /** Benchmark is deterministic (bitwise reproducible): hints skip repeat re-measurement suggestions. */ + deterministic?: boolean; [key: string]: unknown; } diff --git a/external_plugins/autoresearch/mcp/lib/validate.ts b/external_plugins/autoresearch/mcp/lib/validate.ts index f098dd13..f1d09e31 100644 --- a/external_plugins/autoresearch/mcp/lib/validate.ts +++ b/external_plugins/autoresearch/mcp/lib/validate.ts @@ -15,7 +15,7 @@ const VALID_STATUS = new Set([ * A ledger row as the auditor sees it. The audit's whole job is to accept * possibly-malformed rows, so fields are loose (string status/type allowed). */ -export interface LedgerRow { +interface LedgerRow { type?: string; run?: number; segment?: number; @@ -24,7 +24,7 @@ export interface LedgerRow { commit?: string | null; } -export interface LedgerViolation { +interface LedgerViolation { code: string; run?: number; message: string; @@ -49,6 +49,7 @@ export function validateLedger( ): LedgerViolation[] { const violations: LedgerViolation[] = []; const direction = config?.direction ?? "lower"; + let baseline: number | null = null; // first finite metric in this segment let retained: number | null = null; // current best kept metric (null until first keep) let expectedRun = 1; @@ -99,14 +100,32 @@ export function validateLedger( continue; } + // The segment baseline is the first finite measurement (a legacy crash + // row carrying 0 is a placeholder, not a measurement). + if (r.status !== "crash" && baseline == null) baseline = metric; + if (r.status === "keep") { if (retained == null) { - retained = metric; // baseline / first keep + // The first keep establishes the retained value, but it must not be + // worse than the segment baseline the dashboard/state already show. + if ( + baseline != null && + metric !== baseline && + !isBetter(metric, baseline, direction) + ) { + push( + "keep_without_improvement", + r.run, + `first keep metric ${metric} is worse than baseline ${baseline} (${direction})`, + ); + } else { + retained = metric; + } } else if (!isBetter(metric, retained, direction)) { push( "keep_without_improvement", r.run, - `keep metric ${metric} does not beat retained ${retained} (${direction})`, + `keep metric ${metric} does not beat retained ${retained} (${direction}); a keep must improve on the current best — beating the baseline alone is not enough (log this run as discard instead)`, ); } else { retained = metric; diff --git a/external_plugins/autoresearch/mcp/server.ts b/external_plugins/autoresearch/mcp/server.ts index fd0e080e..9bf7d650 100644 --- a/external_plugins/autoresearch/mcp/server.ts +++ b/external_plugins/autoresearch/mcp/server.ts @@ -1,16 +1,23 @@ #!/usr/bin/env node // zcode-autoresearch MCP server (stdio, newline-delimited JSON-RPC). -// Tools: init_experiment / run_experiment / log_experiment / export_dashboard. +// Tools: init_experiment / run_experiment / log_experiment / export_dashboard / clear_experiments. // Design: experiment/autoresearch + ADR-1 (MCP tools carry mechanism). import { spawn } from "node:child_process"; +import type { + ChildProcessByStdio, + ChildProcessWithoutNullStreams, +} from "node:child_process"; +import type { Readable } from "node:stream"; import { appendFileSync, + chmodSync, existsSync, + mkdirSync, readFileSync, + renameSync, statSync, writeFileSync, } from "node:fs"; -import { createHash } from "node:crypto"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { @@ -21,9 +28,18 @@ import { computeConfidence, isBetter, } from "./lib/experiment.ts"; +import { + resolveBash, + killTreeOrWrapper, + quoteShellArg, + spawnEnv, +} from "./lib/bash.ts"; import { autoPaths, appendLedgerEntry, + plateauCalibration, + readLedger, + rebuildHistory, rebuildState, readSessionConfig, writeDashboard, @@ -31,21 +47,34 @@ import { import { commitExperiment, rollbackWorkingTree, + dirtyFiles, isGitRepo, + hasCommit, + ensureSessionStateIgnored, isDirty, currentBranch, + safeShortHash, + headSha, + undoCommitTo, } from "./lib/git.ts"; import { renderDashboard } from "./lib/dashboard.ts"; import { resolveWorkCwd } from "./lib/paths.ts"; import { validateLedger } from "./lib/validate.ts"; +import { + currentHarnessHashes, + diffHarnessEdits, + sha256File, +} from "./lib/harness.ts"; import { ensureDashboardServer, broadcastDashboardUpdate, } from "./lib/dashboard-server.ts"; +import { sweepStaleSpillFiles } from "./lib/spill.ts"; import type { Confidence, Direction, LedgerRun, + PendingObservation, RunStatus, SessionState, } from "./lib/types.ts"; @@ -67,6 +96,8 @@ interface RunOutcome { outputTail: string; logFile: string | null; timedOut: boolean; + /** Set when the command could not be spawned at all (missing bash, resolveBash throw). */ + spawnError?: string; } interface HookOutcome { @@ -85,6 +116,11 @@ interface InitToolArgs { metric_unit?: string; metricUnit?: string; direction?: string; + plateau_min_improvement?: unknown; + plateauMinImprovement?: unknown; + plateau_window?: unknown; + plateauWindow?: unknown; + deterministic?: unknown; } interface RunToolArgs { @@ -104,16 +140,50 @@ interface LogToolArgs { } const projectCwd = process.cwd(); -// Effective research directory: `.auto/config.json` may set workingDir. +// Effective research directory: `.auto/config.json` may set workingDir. An +// invalid workingDir throws here on purpose — the server must fail loudly at +// startup rather than point destructive git operations at the project root. const cwd = resolveWorkCwd(projectCwd); const paths = autoPaths(cwd); -const envMax = Number(process.env.AR_MAX_ITERATIONS); -const DEFAULT_MAX_ITERATIONS = - Number.isFinite(envMax) && envMax > 0 ? envMax : 20; -const BENCHMARK_TIMEOUT_MS = - Number(process.env.AR_BENCHMARK_TIMEOUT_MS) || 600_000; -const CHECKS_TIMEOUT_MS = Number(process.env.AR_CHECKS_TIMEOUT_MS) || 300_000; +// Single source of truth for the reported version: read the plugin manifest +// instead of hardcoding a third copy (plugin.json and marketplace.json are +// the other two, guarded by tests/marketplace-contract.test.ts). +const PLUGIN_VERSION = (() => { + try { + const manifest = JSON.parse( + readFileSync( + new URL("../.zcode-plugin/plugin.json", import.meta.url), + "utf8", + ), + ) as { version?: string }; + return manifest.version ?? "0.0.0"; + } catch { + return "0.0.0"; + } +})(); + +// Numeric env vars are convenience overrides (manifest userConfig seeded via +// env by the host is the primary channel): an invalid value falls back to the +// default with a stderr diagnostic instead of silently taking effect — a +// negative timeout would pass the `||` chain and SIGTERM every run instantly. +function envPositive(name: string, fallback: number): number { + const raw = Number(process.env[name]); + if (Number.isFinite(raw) && raw > 0) return raw; + if (process.env[name] !== undefined && process.env[name] !== "") { + process.stderr.write( + `[autoresearch] ${name}=${process.env[name]} is not a positive finite number; using default ${fallback}\n`, + ); + } + return fallback; +} + +const DEFAULT_MAX_ITERATIONS = envPositive("AR_MAX_ITERATIONS", 20); +const BENCHMARK_TIMEOUT_MS = envPositive("AR_BENCHMARK_TIMEOUT_MS", 600_000); +// Node's setTimeout treats delays above 2^31-1 ms as 1ms: an unclamped huge +// timeout_seconds would SIGTERM the benchmark immediately instead of never. +const MAX_TIMEOUT_MS = 2 ** 31 - 1; +const CHECKS_TIMEOUT_MS = envPositive("AR_CHECKS_TIMEOUT_MS", 300_000); // LLM-facing output budget (mirrors pi-autoresearch): tight truncation. const LLM_MAX_LINES = 10; @@ -155,10 +225,37 @@ function consecutiveFailures() { return Number.isFinite(v) && v > 0 ? v : 3; } +/** Loop calibration (sanitized in ledger.plateauCalibration) for rebuildState. */ +function plateauOpts() { + const cal = plateauCalibration(projectCwd); + return { + plateauWindow: cal.window, + plateauMinImprovement: cal.minImprovement, + deterministic: cal.deterministic, + }; +} + +/** + * Plateau hint wording reflecting the configured window/threshold. A + * deterministic benchmark (bitwise reproducible) cannot be confirmed by + * re-measurement, so the repeat suggestion is dropped for it. + */ +function plateauHintText(): string { + const cal = plateauCalibration(projectCwd); + const window = cal.window ?? 5; + const min = cal.minImprovement ?? 0.01; + const pct = `${(min * 100).toFixed(2).replace(/\.?0+$/, "")}%`; + const confirm = cal.deterministic + ? "" + : "; consider rerunning with repeat:3 to confirm"; + return `plateau detected (last ${window} runs improved < ${pct})${confirm}, opening a new segment, or stopping`; +} + function sessionState() { return rebuildState(cwd, { maxIterations: maxIterations(), consecutiveFailures: consecutiveFailures(), + ...plateauOpts(), }); } @@ -166,24 +263,32 @@ function sessionState() { * Confidence over a segment's runs (design D4): the same aggregation * rebuildState performs (baseline = first valid metric, best = best keep, * MAD over the finite values), kept next to the server so the ledger snapshot - * and the returned confidence share a single computation. + * and the returned confidence share a single computation. A deterministic + * benchmark (bitwise reproducible) has no noise floor to divide by — the + * pooled outcome spread across experiments is effect size, not noise — so + * the delta is exact and no confidence ratio is reported. */ function confidenceForSegment( runs: LedgerRun[], direction: Direction, + deterministic: boolean, ): Confidence | null { let baseline: number | null = null; let best: number | null = null; const values: number[] = []; for (const r of runs) { - if (baseline == null && r.metric != null) baseline = r.metric; + // Crash rows measure nothing (ADR-10; legacy rows carry a 0 placeholder): + // stay out of baseline and the value set, same as rebuildState. + if (baseline == null && r.metric != null && r.status !== "crash") + baseline = r.metric; if (r.status === "keep" && r.metric != null) { if (best == null || isBetter(r.metric, best, direction)) best = r.metric; } - if (r.metric != null && Number.isFinite(r.metric)) values.push(r.metric); + if (r.metric != null && Number.isFinite(r.metric) && r.status !== "crash") + values.push(r.metric); } if (values.length === 0) return null; - return computeConfidence({ values, baseline, best }); + return computeConfidence({ values, baseline, best, deterministic }); } // Revisit nudge (design D2): the standing check that keeps a discard from @@ -191,6 +296,12 @@ function confidenceForSegment( const REVISIT_NUDGE = "revisit check: before choosing the next experiment, check whether the latest result invalidates a previous discard's rollback reason (see the asi.rollback field on the discard/checks_failed rows in the ledger); if yes: name what changed and weigh a targeted retry of that discard (annotate it with asi.revisits_run: ); if no: move on; verification reruns for measurement noise are separate and do not carry the annotation."; +// noop rolls nothing back by design, so a dirty tree would silently ride into +// the next measurement. Advisory (the dirt may be legitimate user files): +// surface it, let the agent decide. +const NOOP_DIRTY_WARNING = + "working tree has uncommitted changes outside session state; noop rolls nothing back, so these changes will be part of the next measurement. Restore them (or log them as an experiment) if they are not meant to be measured."; + function truncateTail( text: unknown, maxLines = LLM_MAX_LINES, @@ -211,17 +322,39 @@ function truncateTail( // bounded tail survive in memory. const SPILL_THRESHOLD_BYTES = 2 * 1024 * 1024; const TAIL_CAP_BYTES = 64 * 1024; +// Spilled logs are read at most until the tool call returns its tail; past +// this age they are only temp-dir litter (and whatever the benchmark printed). +const SPILL_MAX_AGE_MS = 7 * 24 * 3600 * 1000; const MAX_METRIC_LINES = 1000; const KILL_GRACE_MS = 5_000; function runCommand(command: string, timeoutMs: number): Promise { return new Promise((resolve) => { const started = Date.now(); - const proc = spawn("bash", ["-c", command], { - cwd, - detached: true, // own process group so we can kill the tree - stdio: ["ignore", "pipe", "pipe"], - }); + let proc: ChildProcessByStdio; + try { + const bashPath = resolveBash(); + proc = spawn(bashPath, ["-c", command], { + cwd, + detached: true, // own process group so we can kill the tree + stdio: ["ignore", "pipe", "pipe"], + env: spawnEnv(bashPath), + }); + } catch (err) { + // resolveBash throws when no usable bash exists (e.g. Windows without + // Git Bash) — surface it instead of leaving the tool call hanging. + resolve({ + exitCode: null, + signal: null, + durationMs: 0, + output: "", + outputTail: "", + logFile: null, + timedOut: false, + spawnError: err instanceof Error ? err.message : String(err), + }); + return; + } const chunks: Buffer[] = []; // full output, only while under the threshold let totalBytes = 0; let logFile: string | null = null; @@ -236,8 +369,9 @@ function runCommand(command: string, timeoutMs: number): Promise { if (totalBytes > SPILL_THRESHOLD_BYTES) { logFile = join( tmpdir(), - `pi-experiment-${process.pid}-${Date.now()}.log`, + `autoresearch-experiment-${process.pid}-${Date.now()}.log`, ); + sweepStaleSpillFiles(tmpdir(), SPILL_MAX_AGE_MS, Date.now()); writeFileSync(logFile, Buffer.concat(chunks)); chunks.length = 0; log("output overflowed, spilling to", logFile); @@ -267,16 +401,39 @@ function runCommand(command: string, timeoutMs: number): Promise { carry = lines.pop() ?? ""; if (carry.length > TAIL_CAP_BYTES) carry = carry.slice(-TAIL_CAP_BYTES); for (const line of lines) { - if (metricLines.length < MAX_METRIC_LINES && line.startsWith("METRIC ")) - metricLines.push(line); + // Trim before matching: the non-spill path (parseMetricLines) accepts + // indented " METRIC x=1" lines, and the two paths must agree or a + // >2MB output silently loses a primary metric a small output kept. + const trimmed = line.trim(); + if ( + metricLines.length < MAX_METRIC_LINES && + trimmed.startsWith("METRIC ") + ) + metricLines.push(trimmed); } }; proc.stdout.on("data", onData); proc.stderr.on("data", onData); + // "close" waits for stdio to drain; a detached descendant holding the + // pipes delays it indefinitely — an MSYS binary that survived the Windows + // tree walk (taskkill kills the bash wrapper but not every forked child), + // or a daemonized grandchild outside the process group the POSIX kill + // addresses. The buffer already holds everything needed to judge the + // run, so exit arms a bounded fallback in ALL cases (timeout-kill and + // normal exit alike); close pre-empts when it fires and resolve is + // idempotent, so a late close is a no-op. let didTimeout = false; let killTimer: NodeJS.Timeout | null = null; + let fallbackTimer: NodeJS.Timeout | null = null; const kill = () => { didTimeout = true; + if (process.platform === "win32") { + // Windows cannot address process groups via negative pids (ESRCH); + // taskkill /T /F walks and kills the whole child tree instead. If the + // walk fails, kill the wrapper so the timeout still settles. + killTreeOrWrapper(proc.pid, () => proc.kill()); + return; + } try { if (proc.pid != null) process.kill(-proc.pid, "SIGTERM"); } catch { @@ -293,9 +450,27 @@ function runCommand(command: string, timeoutMs: number): Promise { }, KILL_GRACE_MS); }; const timer = setTimeout(kill, timeoutMs); - proc.on("close", (code, signal) => { + // A spawn failure (bash missing, cwd gone) otherwise never fires + // "close" and would hang the tool call — resolve it as spawnError. + proc.on("error", (err) => { + clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + if (fallbackTimer) clearTimeout(fallbackTimer); + resolve({ + exitCode: null, + signal: null, + durationMs: Date.now() - started, + output: "", + outputTail: "", + logFile: null, + timedOut: false, + spawnError: err.message, + }); + }); + const finalize = (code: number | null, signal: NodeJS.Signals | null) => { clearTimeout(timer); if (killTimer) clearTimeout(killTimer); + if (fallbackTimer) clearTimeout(fallbackTimer); const elapsed = Date.now() - started; const full = Buffer.concat(chunks).toString("utf8"); resolve({ @@ -307,14 +482,40 @@ function runCommand(command: string, timeoutMs: number): Promise { logFile, timedOut: didTimeout, }); + }; + proc.on("close", (code, signal) => finalize(code, signal)); + proc.on("exit", (code, signal) => { + // Bounded settle: give close a short grace to deliver the drained + // output, then finalize from the exit facts. Without this, an + // exit-then-no-close process (pipes held by a detached descendant) + // hangs the tool call forever — the kill timer fires into nothing and + // no further event ever re-arms finalize. + if (fallbackTimer) return; + fallbackTimer = setTimeout(() => finalize(code, signal), KILL_GRACE_MS); }); }); } async function runChecks(checksFile: string) { - const res = await runCommand(`bash ${checksFile}`, CHECKS_TIMEOUT_MS); + // The path goes through `bash -c`, where backslashes are escape characters — + // a Windows absolute path would be eaten into a nonexistent one (exit 127, + // read as "checks failed"). Forward slashes are valid on both platforms; + // quoteShellArg keeps apostrophes in the project path from ending the quote. + const posix = checksFile.replaceAll("\\", "/"); + const res = await runCommand( + `bash ${quoteShellArg(posix)}`, + CHECKS_TIMEOUT_MS, + ); + // A verdict exists only when the script ran to completion: exit 0 = pass, + // 1..125 = deliberate fail. 126/127 (not executable / not found), null + // (killed by signal) and large fatal codes (Windows 0xC000013A) mean the + // gate never produced a judgment. Those are infra, not correctness + // failures, and must not block keeps — a broken interpreter would + // otherwise turn every checks run into a veto (evidence 2026-10-28). + const verdict = res.exitCode !== null && res.exitCode < 126; return { - failed: res.exitCode !== 0, + verdict, + failed: verdict && res.exitCode !== 0, exitCode: res.exitCode, durationMs: res.durationMs, outputTail: truncateTail(res.outputTail, 80, 4096), @@ -330,6 +531,10 @@ const HOOK_MAX_BYTES = 8 * 1024; function isExecutable(file: string): boolean { if (!existsSync(file)) return false; + // NTFS carries no Unix exec bits: Node's stat mode is always 0666, so the + // bit test below would make iteration hooks unreachable on Windows. The + // spawn goes through bash anyway, so existence is the opt-in there. + if (process.platform === "win32") return true; try { return (statSync(file).mode & 0o111) !== 0; } catch { @@ -338,11 +543,6 @@ function isExecutable(file: string): boolean { } // --- benchmark drift detection (frozen-file hashes) ------------------------- -function sha256File(file: string): string | null { - if (!existsSync(file)) return null; - return createHash("sha256").update(readFileSync(file)).digest("hex"); -} - function currentBenchmarkHashes(): { measure: string | null; checks: string | null; @@ -363,15 +563,49 @@ function readBenchmarkHashes() { /** Merge a patch into the project's `.auto/config.json` (creates if missing). */ function patchSessionConfig(patch: Record): void { - const cfgPath = join(projectCwd, ".auto", "config.json"); + const cfgDir = join(projectCwd, ".auto"); + const cfgPath = join(cfgDir, "config.json"); + mkdirSync(cfgDir, { recursive: true }); let cfg: Record = {}; try { cfg = JSON.parse(readFileSync(cfgPath, "utf8")) as Record; - } catch { - /* start fresh */ + } catch (err) { + // ENOENT → start fresh. A file that exists but is not valid JSON must fail + // loudly: silently starting fresh would make the write below wipe + // workingDir, harnessHashes and the off flag along with the corruption. + if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") { + throw new Error( + `.auto/config.json exists but is not valid JSON (${err instanceof Error ? err.message : String(err)}); fix or remove it — refusing to overwrite the session config.`, + ); + } } Object.assign(cfg, patch); - writeFileSync(cfgPath, JSON.stringify(cfg, null, 2)); + writeSessionConfigAtomic(cfgPath, cfg); +} + +/** + * config.json is the cross-call state family (calibration keys, maxIterations, + * auditBypass sit next to server-managed keys) and readSessionConfig degrades + * a corrupt file to `{}` — an in-place writeFileSync truncated by a crash + * mid-write would silently reset every override on next read. Write to a + * sibling temp file and rename: rename is atomic on POSIX, so readers only + * ever see the old or the new complete file. + */ +function writeSessionConfigAtomic( + cfgPath: string, + cfg: Record, +): void { + const tmpPath = `${cfgPath}.tmp`; + writeFileSync(tmpPath, JSON.stringify(cfg, null, 2)); + // rename(2) replaces the target regardless of its permission bits — carry + // the existing mode over, or a chmod-restricted config silently gains + // default permissions on every update. + try { + chmodSync(tmpPath, statSync(cfgPath).mode); + } catch { + // ENOENT (first write) → keep the tmp file's default mode. + } + renameSync(tmpPath, cfgPath); } function writeBenchmarkHashes(hashes: { @@ -381,10 +615,41 @@ function writeBenchmarkHashes(hashes: { patchSessionConfig({ benchmarkHashes: hashes }); } +/** + * Clear every `pending*` key from `.auto/config.json` (the server cross-call + * state family: pendingChecksFailed, pendingObservation). Segment boundaries + * (init_experiment / clear_experiments) reset the whole prefix instead of a + * per-field list: pendingObservation was added later and both hand-maintained + * lists missed it, letting a previous session's unconsumed observation attest + * run #1 of the new segment (red test 2026-11-07). The `pending` prefix is + * the contract (SessionConfig in types.ts); future keys are covered + * automatically. A missing or unparsable config clears nothing and is not + * created: unparsable JSON cannot carry a consumable snapshot anyway + * (readSessionConfig treats it as empty), and clear must not gain a new + * failure mode. + */ +function resetPendingState(): void { + const cfgPath = join(projectCwd, ".auto", "config.json"); + let cfg: Record; + try { + cfg = JSON.parse(readFileSync(cfgPath, "utf8")) as Record; + } catch { + return; + } + let touched = false; + for (const key of Object.keys(cfg)) { + if (key.startsWith("pending")) { + delete cfg[key]; + touched = true; + } + } + if (touched) writeSessionConfigAtomic(cfgPath, cfg); +} + /** * Persist the checks outcome of the latest run_experiment so log_experiment's * keep gate works even across an MCP server restart (pi `runtime.lastRunChecks` - * equivalent, but on disk). `failed` is true only when checks ran and failed. + * equivalent, but on disk). `failed` is true only when checks produced a verdict and it was negative; a checks script that never produced a verdict (not executed / killed) is infra and must not veto keeps. */ function setPendingChecksFailed(failed: boolean): void { patchSessionConfig({ pendingChecksFailed: failed }); @@ -394,6 +659,22 @@ function pendingChecksFailed(): boolean { return readSessionConfig(projectCwd).pendingChecksFailed === true; } +/** + * Observation snapshot of the latest run_experiment (see PendingObservation): + * run_experiment is the loop's only observer, and before this snapshot its + * observation died with the response — everything quantitative in the ledger + * was agent transcription. Same persistence mechanics as pendingChecksFailed + * (config.json, restart-survivable); overwritten by the next run, consumed by + * every successful log. + */ +function setPendingObservation(o: PendingObservation | null): void { + patchSessionConfig({ pendingObservation: o }); +} + +function readPendingObservation(): PendingObservation | null { + return readSessionConfig(projectCwd).pendingObservation ?? null; +} + /** * Compare current frozen-file hashes against the session's recorded ones. * Returns { drift, reason, deleted } where drift is true when a recorded hash @@ -438,6 +719,42 @@ function checkBenchmarkDrift() { }; } +// --- mandatory harness trace (editable-surface hashes) ----------------------- +// The edit step's discipline (edit rows, verdicts) is advisory; the FACT of a +// surface change is not. Each accounting checkpoint diffs the editable +// surface against the config baseline and appends one harness_edit row per +// moved file — including hand edits, which no tool hook could see. Sessions +// without a baseline (pre-trace config.json) get one silently, no rows. +function recordHarnessEdits(segment?: number): void { + try { + const current = currentHarnessHashes(cwd); + const baseline = readSessionConfig(projectCwd).harnessHashes; + if (!baseline) { + patchSessionConfig({ harnessHashes: current }); + return; + } + const seg = segment ?? sessionState().segment; + for (const file of diffHarnessEdits(baseline, current)) { + appendLedgerEntry(cwd, { + type: "harness_edit", + ts: new Date().toISOString(), + segment: seg, + file, + digest_before: baseline[file] ?? null, + digest_after: current[file] ?? null, + }); + } + patchSessionConfig({ harnessHashes: current }); + } catch (err) { + // Tracing is bookkeeping, not a gate: a failed trace must never block + // the run/log/export it rides on — but it must not fail silently either + // (never-silent rule). + process.stderr.write( + `[autoresearch] harness trace recording failed (bookkeeping only, not blocking): ${err instanceof Error ? err.message : String(err)}\n`, + ); + } +} + /** * Run an iteration hook: bash diff --git a/external_plugins/autoresearch/mcp/lib/ledger.ts b/external_plugins/autoresearch/mcp/lib/ledger.ts index d85e9f59..eafa7fb5 100644 --- a/external_plugins/autoresearch/mcp/lib/ledger.ts +++ b/external_plugins/autoresearch/mcp/lib/ledger.ts @@ -230,6 +230,7 @@ export interface SessionHistory { runs: LedgerRun[]; totalRuns: number; kept: number; + segmentCount: number; firstMetric: number | null; bestMetric: number | null; lastRunAt: string | null; @@ -251,7 +252,14 @@ export function rebuildHistory(entries: LedgerEntry[]): SessionHistory { if (e.type === "config") { config = e; } else if (e.type === "run") { - runs.push(e); + // Hand-written ledgers may omit a run's segment: attribute it to the + // latest config row seen so far (projection-only normalization; the + // ledger itself is never rewritten). + runs.push( + e.segment === undefined && config + ? { ...e, segment: config.segment } + : e, + ); // Crash rows measure nothing (see rebuildState): a legacy 0-placeholder // must not become the dashboard's global baseline. if ( @@ -292,6 +300,9 @@ export function rebuildHistory(entries: LedgerEntry[]): SessionHistory { runs, totalRuns: runs.length, kept: runs.filter((r) => r.status === "keep").length, + // Distinct segments among actual runs (config-only segments have no + // trajectory to bound); drives the trend chart's boundary annotations. + segmentCount: new Set(runs.map((r) => r.segment)).size, firstMetric, bestMetric, lastRunAt, diff --git a/external_plugins/autoresearch/skills/autoresearch/SKILL.md b/external_plugins/autoresearch/skills/autoresearch/SKILL.md index 27bc292c..f0f7e502 100644 --- a/external_plugins/autoresearch/skills/autoresearch/SKILL.md +++ b/external_plugins/autoresearch/skills/autoresearch/SKILL.md @@ -60,7 +60,7 @@ Full protocol: `references/loop-protocol.md` → Harness editing. ## Dashboard -Run `export_dashboard` (or `/autoresearch:export`) to start a **live** local dashboard at a `127.0.0.1` URL (auto-refreshes via SSE after every experiment); it also writes `autoresearch-dashboard.html` as a static fallback. Open the URL with a browser (e.g. `open `) to watch progress. Reset the session with `clear_experiments` (or `/autoresearch:clear`); when experiments are done, `/autoresearch:finalize` splits kept work into clean PR-able branches. +Run `export_dashboard` (or `/autoresearch:export`) to start a **live** local dashboard at a `127.0.0.1` URL (auto-refreshes via SSE after every experiment); it also writes `autoresearch-dashboard.html` as a static **snapshot** — rewritten only on the next export, never per round; the live page is the real-time view. **Open the URL once for the user** right after the first successful export (`open ` on macOS, `xdg-open ` on Linux, `cmd /c start ` on Windows) — once, not per round; SSE keeps the page current. Skip the open command when running headless (SSH / CI / remote executor — the browser would open where the user cannot see it) or when the user declined watching; hand over the URL in text instead. At the loop's final export, do not open the browser out of nowhere if it was never opened this session — report the URL and the static path. Reset the session with `clear_experiments` (or `/autoresearch:clear`); when experiments are done, `/autoresearch:finalize` splits kept work into clean PR-able branches. ## Working directory diff --git a/external_plugins/autoresearch/skills/autoresearch/references/loop-protocol.md b/external_plugins/autoresearch/skills/autoresearch/references/loop-protocol.md index 5e6384d2..7b0b13fb 100644 --- a/external_plugins/autoresearch/skills/autoresearch/references/loop-protocol.md +++ b/external_plugins/autoresearch/skills/autoresearch/references/loop-protocol.md @@ -56,7 +56,7 @@ The prefixes are an advisory convention: the tools never validate them, and unta - User interrupts → complete the current run+log cycle first, then summarize. - **Do not** run experiments that change `.auto/measure.sh` or `.auto/checks.sh`; if the target changes, call `init_experiment` again (new segment) instead. - Clear the ledger with `clear_experiments` (`/autoresearch:clear`; the other `.auto/` files are kept). -- When done, run `export_dashboard` for a live URL + static report, then `/autoresearch:finalize` to split kept experiments into topic branches. +- When done, run `export_dashboard` for a live URL + static report — the final export is the report archive: if the browser was never opened this session, do not pop it open now; hand over the URL and the static `autoresearch-dashboard.html` path in text. Then `/autoresearch:finalize` splits kept experiments into topic branches. ## Harness editing (second-order loop) diff --git a/external_plugins/autoresearch/tests/dashboard.test.ts b/external_plugins/autoresearch/tests/dashboard.test.ts index 9bc7a67d..395f7efa 100644 --- a/external_plugins/autoresearch/tests/dashboard.test.ts +++ b/external_plugins/autoresearch/tests/dashboard.test.ts @@ -311,12 +311,12 @@ test("live server survives a deleted ledger (404) and dead SSE clients", async ( }); }); -test("renderer: result-first hero metrics, dual theme, overflow guards, neutral no-op", () => { +test("renderer: console hero with KPI cards, dark-first tokens, neutral no-op", () => { const html = renderDashboard(sampleHistory()); assert.match(html, /box-sizing: border-box/); - // no eyebrow: the serif session title opens the page directly - assert.doesNotMatch(html, /EXPERIMENT ARCHIVE/); - assert.match(html, /prefers-color-scheme: dark/); + // dark-first: deep-navy token set is the default, light re-maps via media query + assert.match(html, /color-scheme: dark/); + assert.match(html, /prefers-color-scheme: light/); assert.match(html, /overflow-wrap: anywhere/); // result composition: baseline → best with the overall delta in accent assert.match(html, /class="result"/); @@ -326,18 +326,32 @@ test("renderer: result-first hero metrics, dual theme, overflow guards, neutral ); assert.match(html, /
40<\/div>
best/); assert.match(html, /
−60%<\/div>/); - // meta line: process counts + confidence verdict, no band grid anywhere - assert.match(html, /class="meta"/); + // KPI cards carry the process counts (NO-OP not counted as reverted) + assert.match(html, /class="kpis"/); + assert.match( + html, + /
5<\/div>
实验总数<\/div>/, + ); + assert.match( + html, + /
3<\/div>
保留<\/div>/, + ); assert.match( html, - /5 experiments · <\/span>3 kept · <\/span>1 reverted · <\/span>1 no-op/, + /
1<\/div>
回退<\/div>/, ); - assert.match(html, /PASS<\/span> 2\.67× noise/); + assert.match( + html, + /
1<\/div>
无改动<\/div>/, + ); + // meta line: confidence verdict only (counts moved to KPI cards) + assert.match(html, /class="meta"/); + assert.match(html, /通过<\/span> 2\.67× 噪声/); assert.doesNotMatch(html, /class="band"/); assert.match(html, /class="ledger"/); // no-op is neutral, never the crash-red fallback - assert.match(html, /r-stamp noop">NO-OP/); - assert.doesNotMatch(html, /r-stamp crash">NO-OP/); + assert.match(html, /r-stamp noop">无改动/); + assert.doesNotMatch(html, /r-stamp crash">无改动/); }); test("renderer: SVG trend line across the ledger with baseline reference", () => { @@ -356,6 +370,73 @@ test("renderer: SVG trend line across the ledger with baseline reference", () => assert.match(html, />100<\/text>/); assert.match(html, /class="dk"/); // keep points filled assert.match(html, /class="dh"/); // non-keep points hollow + // no per-point ordinal text; X axis is a sparse "全局轮次" integer scale + assert.doesNotMatch(html, /class="dv"/); + assert.match(html, /class="xt"/); + assert.match(html, /全局轮次/); + // named legend + semantics note + summary strip under the chart + assert.match(html, /轨迹连线<\/i>保留/); + assert.match(html, /实心为保留 · 空心为未保留 · 虚线为分段边界/); + assert.match( + html, + /
分段数 1<\/b><\/span>轮次数 4<\/b><\/span>保留数 3<\/b><\/span>首轮→最优 −60<\/b><\/span><\/div>/, + ); + // tooltip data attributes carry real ledger values on each point anchor + assert.match( + html, + / { + const runs = Array.from({ length: 30 }, (_, i) => + i % 3 === 2 + ? (["discard", 100 + (i % 5)] as RunTuple) + : (["keep", 90 + (i % 4)] as RunTuple), + ); + const html = renderDashboard(sampleHistory(runs)); + const xt = html.match(/class="xt"/g) ?? []; + // 30 runs → step 5 → ticks 5..30 → 6 per variant, ≤8 by design + assert.ok(xt.length >= 4 && xt.length <= 16, `x tick count ${xt.length}`); + assert.doesNotMatch(html, /class="dv"/); + assert.ok((html.match(/ { + const one = rebuildHistory(entriesWithRuns([["keep", 100]])); + assert.equal(one.segmentCount, 1); + const two = rebuildHistory([ + { + type: "config", + segment: 1, + name: "a", + metricName: "m", + direction: "lower", + }, + { type: "run", run: 1, status: "keep", metric: 100, description: "" }, + { + type: "config", + segment: 2, + name: "b", + metricName: "m", + direction: "lower", + }, + { type: "run", run: 1, status: "keep", metric: 90, description: "" }, + // a config-only segment has no runs → not counted + { + type: "config", + segment: 3, + name: "c", + metricName: "m", + direction: "lower", + }, + ]); + assert.equal(two.segmentCount, 2); }); test("renderer: no trend below two valid points", () => { @@ -366,8 +447,72 @@ test("renderer: no trend below two valid points", () => { ["crash", null], ]), ); + // no trend chart (page icons are svg, the chart is not) + assert.doesNotMatch(html, /无改动/); +}); + +test("renderer: empty ledger shows the empty state without KPI cards", () => { + const html = renderDashboard(rebuildHistory([])); + assert.match(html, /class="empty"/); + assert.doesNotMatch(html, /class="kpis"/); assert.doesNotMatch(html, /NO-OP/); + assert.doesNotMatch(html, /class="result"/); +}); + +test("renderer: KPI card counts follow the real ledger", () => { + const html = renderDashboard( + sampleHistory([ + ["keep", 100], + ["keep", 90], + ["discard", 120], + ["crash", null], + ["checks_failed", null], + ["noop", null], + ["noop", null], + ]), + ); + // total 7 · kept 2 · reverted 3 (discard+crash+checks_failed) · no-op 2 + assert.match( + html, + /
7<\/div>
实验总数<\/div>/, + ); + assert.match( + html, + /
2<\/div>
保留<\/div>/, + ); + assert.match( + html, + /
3<\/div>
回退<\/div>/, + ); + assert.match( + html, + /
2<\/div>
无改动<\/div>/, + ); +}); + +test("renderer: long descriptions render full text in a clamp line (no details)", () => { + // 2026-11-13: one clamp language for every long value — no
, no + // preview duplication; JS clamps overflowing items to one faded line and + // expands on click, without JS the full text stays readable. + const entries = entriesWithRuns([ + ["keep", 100], + ["keep", 90], + ]); + const runs = entries.filter((e) => e.type === "run") as LedgerRun[]; + const long = "把热点路径从同步 fs 写改为批量异步写，".repeat(20); + runs[0].description = long; + runs[1].description = "short one"; + const html = renderDashboard(rebuildHistory(entries)); + // no details/summary anywhere; full text present exactly once + assert.doesNotMatch(html, /
/); + assert.match(html, /short one<\/span>/); + assert.match(renderLiveDashboard(rebuildHistory(entries)), /clamp-line/); }); test("renderer: delta of exactly zero renders neutral ±0 (no arrow mismatch)", () => { @@ -417,19 +562,22 @@ test("renderer: mid-ledger config rows never split the flat ledger", () => { ]; const history = rebuildHistory(entries); const html = renderDashboard(history); - // all runs render in one flat ledger — no segment grouping anywhere + // all runs render in one flat ledger — no list-level grouping anywhere assert.equal((html.match(/class="run"/g) ?? []).length, 3); - assert.doesNotMatch(html, /SEGMENT/); assert.doesNotMatch(html, /seg 0/); assert.doesNotMatch(html, /seg-div/); assert.doesNotMatch(html, /class="bound"/); + // the trend chart marks the segment boundary (fact annotation, not grouping) + assert.match(html, /class="seg-line"/); + assert.match(html, /分段 2<\/text>/); + assert.match(html, /分段数 2<\/b>/); // title follows the latest config assert.match(html, /

phase-two<\/h1>/); // hero stats are global: baseline 100, best 80 assert.match(html, /
100<\/div>
baseline/); // Δ is computed against the global baseline (100): run #2 (90) → ▲ +10 assert.match(html, /▲ \+10<\/span>/); - assert.match(html, /Δ vs baseline/); + assert.match(html, /Δ 相对基线/); // global best 80 → overall improvement shows in the hero stat assert.match(html, /class="n">−20%<\/div>
overall/); }); @@ -445,15 +593,12 @@ test("renderer: footer pins the git hash when provided and dates the export", () const runs = entries.filter((e) => e.type === "run") as LedgerRun[]; runs[1].timestamp = "2026-10-27T15:07:00.000Z"; // last run drives the date const history = rebuildHistory(entries); - assert.match( - renderDashboard(history, { pin: "abc1234" }), - /pinned at abc1234/, - ); - assert.match(renderDashboard(history), /10 runs|2 runs/); - assert.doesNotMatch(renderDashboard(history), /pinned at/); + assert.match(renderDashboard(history, { pin: "abc1234" }), /锚定 abc1234/); + assert.match(renderDashboard(history), /10 次实验|2 次实验/); + assert.doesNotMatch(renderDashboard(history), /锚定/); const html = renderDashboard(history, { pin: "abc1234" }); assert.match(html, /autoresearch · 2026-10-27/); - assert.match(html, /2 runs · pinned at abc1234/); + assert.match(html, /2 次实验 · 锚定 abc1234/); }); test("rebuildHistory: flat runs, latest-config direction, leading runs tolerated", () => { @@ -514,51 +659,11 @@ test("renderer: revisits_run marker renders only for positive integers", () => { assert.doesNotMatch(html, /Revisiting #-1/); assert.doesNotMatch(html, /Revisiting #2/); // the rest of the marked row renders unchanged - assert.match(html, /↻ Revisiting #3<\/span>/); + assert.match(html, /↻ Revisiting #3<\/span>/); // the live dashboard shares the same render path assert.match(renderLiveDashboard(rebuildHistory(entries)), /↻ Revisiting #3/); }); -test("renderer: asi hypothesis/next/rollback render as a collapsible block", () => { - // 2026-11-11: the loop's reasoning (hypothesis / next_action_hint / - // rollback) only reached the agent via memory-inject; the dashboard showed - // nothing — a human reading the trend could not answer "why this round". - const entries = entriesWithRuns([ - ["keep", 100], - ["discard", 105], - ]); - const runs = entries.filter((e) => e.type === "run") as LedgerRun[]; - runs[0].asi = { - hypothesis: "swap to catboost", - next_action_hint: "tune depth", - rollback: "kept as winner", - }; - const html = renderDashboard(rebuildHistory(entries)); - assert.match(html, /
假设与结论<\/summary>/); - assert.match( - html, - /假设<\/span>swap to catboost<\/span>/, - ); - assert.match( - html, - /下一步<\/span>tune depth<\/span>/, - ); - assert.match(html, /回滚理由<\/span>/); - // the live dashboard shares the same render path - assert.match(renderLiveDashboard(rebuildHistory(entries)), /class="r-asi"/); - - // rows without asi fields stay block-free - const plain = entriesWithRuns([["keep", 100]]); - const plainHtml = renderDashboard(rebuildHistory(plain)); - assert.doesNotMatch(plainHtml, /class="r-asi"/); - - // ledger text is data, never markup - runs[1].asi = { hypothesis: '' }; - const hostile = renderDashboard(rebuildHistory(entries)); - assert.match(hostile, /<img src=x/); - assert.doesNotMatch(hostile, / { for (const [direction, best, text] of [ ["lower", -20, "−100%"], diff --git a/external_plugins/autoresearch/tests/ledger.test.ts b/external_plugins/autoresearch/tests/ledger.test.ts index b1680680..23c8446a 100644 --- a/external_plugins/autoresearch/tests/ledger.test.ts +++ b/external_plugins/autoresearch/tests/ledger.test.ts @@ -419,3 +419,26 @@ test("rebuildState suppresses confidence for a deterministic benchmark", () => { const exact = rebuildState(cwd, { deterministic: true }); assert.equal(exact.confidence, null); }); + +test("rebuildHistory attributes segment-less runs to the latest config (projection-only)", () => { + const history = rebuildHistory([ + // Before any config row there is nothing to attribute to: the run keeps + // its absent segment instead of being pinned to a guessed one. + { type: "run", run: 1, status: "keep", metric: 100, description: "" }, + { + type: "config", + segment: 1, + name: "a", + metricName: "m", + direction: "lower", + }, + // Hand-written rows may omit segment: attributed to the latest config. + { type: "run", run: 1, status: "keep", metric: 90, description: "" }, + { type: "run", run: 2, status: "keep", metric: 85, description: "" }, + ]); + assert.equal(history.runs[0].segment, undefined); + assert.equal(history.runs[1].segment, 1); + assert.equal(history.runs[2].segment, 1); + // Projection-scope only: the normalization never rewrites the ledger file + // (see the attribution comment in rebuildHistory). +});