diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5778ed9b7..c63653c00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,8 +27,9 @@ name: build # * merging still cannot deploy anything — a deployment pins a digest this # workflow reports, in a separate, operator-approved step # -# Images are signed here as they are in a release. A build that reaches -# production unsigned would be a worse hole than the one this closes. +# Images are not signed, here or in a release: signing was dropped (operator +# decision, 2026-10-20). A digest is trusted by its registry name and its +# SBOM and provenance attestations. # # One branch builds: `main` → `sha-`. Every rule below is an explicit # equality on the ref, never a negation, so adding a branch must mean adding a @@ -133,6 +134,10 @@ jobs: contents: read packages: write id-token: write + # The merged image derives its core from this one by digest, so the + # digest has to leave this job rather than being re-derived from a tag. + outputs: + digest: ${{ steps.push.outputs.digest }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -194,7 +199,7 @@ jobs: # that digest, so what is tested is what was pushed. A failed smoke # leaves an untagged orphan; ghcr-retention.yml prunes untagged # versions after the pinned policy's grace period (7 days). Nothing is - # tagged or signed until the smoke passes, and the layer cache is + # tagged until the smoke passes, and the layer cache is # written once, here. # # v0.1.0 shipped an image whose entrypoint did not exist and v0.1.1 one @@ -249,17 +254,6 @@ jobs: docker buildx imagetools create --prefer-index=false \ -t "${IMAGE}:sha-${GITHUB_SHA:0:7}" "${IMAGE}@${DIGEST}" - - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac - - - name: sign the image - env: - DIGEST: ${{ steps.push.outputs.digest }} - run: | - set -euo pipefail - # The digest, never a tag: a tag can be moved after signing, so a - # signature over a tag says nothing about what gets pulled. - cosign sign --yes "${IMAGE}@${DIGEST}" - - name: report what to pin env: DIGEST: ${{ steps.push.outputs.digest }} @@ -287,7 +281,6 @@ jobs: permissions: contents: read packages: write - id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -340,9 +333,11 @@ jobs: docker run -d --name voice-smoke "$CANDIDATE" trap 'docker logs voice-smoke || true; docker rm -f voice-smoke >/dev/null 2>&1 || true' EXIT - # Wait for the baked models to load — /health gates on it. + # Wait for the baked models to load — /health gates on it. Up to + # 6 min: the ci-runner-1 runners load the models far slower than + # node-f and timed out at the old 2 min (2026-10-23). ready="" - for _ in $(seq 1 60); do + for _ in $(seq 1 180); do if docker exec voice-smoke curl -fsS http://127.0.0.1:8000/health >/dev/null 2>&1; then ready=1; break fi @@ -380,15 +375,6 @@ jobs: cache-from: type=registry,ref=${{ env.CACHE_IMAGE }}:voice-${{ github.ref_name }} cache-to: type=registry,ref=${{ env.CACHE_IMAGE }}:voice-${{ github.ref_name }},mode=max - - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac - - - name: sign the image - env: - DIGEST: ${{ steps.push.outputs.digest }} - run: | - set -euo pipefail - cosign sign --yes "${VOICE_IMAGE}@${DIGEST}" - demo-image: name: build, smoke, and publish the static public demo # The normal stack runs first and exports its web-build layer. This target @@ -401,7 +387,6 @@ jobs: permissions: contents: read packages: write - id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -440,7 +425,7 @@ jobs: build-args: | VOGT_SOURCE_REF=main VOGT_SOURCE_SHA=${{ github.sha }} - VOGT_PRODUCT_VERSION=0.7.17 + VOGT_PRODUCT_VERSION=0.7.22 cache-from: | type=registry,ref=${{ env.CACHE_IMAGE }}:stack-main type=registry,ref=${{ env.CACHE_IMAGE }}:demo-main @@ -501,7 +486,7 @@ jobs: .catch((e) => { console.error(e.message); process.exit(1); }); ' - # A dedicated mobile hostname runs the same signed image with only + # A dedicated mobile hostname runs the same image with only # its root document changed. Prove that mode independently: `/` # must be the showcase while `/index.html` stays the PWA embedded by # its frame (rather than recursively loading the showcase). @@ -556,7 +541,7 @@ jobs: build-args: | VOGT_SOURCE_REF=main VOGT_SOURCE_SHA=${{ github.sha }} - VOGT_PRODUCT_VERSION=0.7.17 + VOGT_PRODUCT_VERSION=0.7.22 provenance: true sbom: true cache-from: | @@ -564,13 +549,6 @@ jobs: type=registry,ref=${{ env.CACHE_IMAGE }}:demo-main cache-to: type=registry,ref=${{ env.CACHE_IMAGE }}:demo-main,mode=max - - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac - - - name: sign the static demo image - env: - DIGEST: ${{ steps.demo-push.outputs.digest }} - run: cosign sign --yes "${DEMO_IMAGE}@${DIGEST}" - - name: report the demo digest to pin env: DIGEST: ${{ steps.demo-push.outputs.digest }} @@ -578,7 +556,7 @@ jobs: { echo "### Static public demo from \`${GITHUB_SHA:0:7}\`" echo - echo "Pin this signed digest in the demo deployment:" + echo "Pin this digest in the demo deployment:" echo echo '```yaml' echo "image: ${DEMO_IMAGE}@${DIGEST}" @@ -619,7 +597,6 @@ jobs: permissions: contents: read packages: write - id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -708,7 +685,7 @@ jobs: --set "*.args.POD_BASE_IMAGE=${POD_BASE_IMAGE}" --set "*.args.VOGT_SOURCE_REF=${VOGT_SOURCE_REF}" --set "*.args.VOGT_SOURCE_SHA=${VOGT_SOURCE_SHA}" - --set "*.args.VOGT_PRODUCT_VERSION=0.7.17" + --set "*.args.VOGT_PRODUCT_VERSION=0.7.22" ) # One label per line. A value is a url, a revision or a timestamp. while IFS= read -r pair || [ -n "$pair" ]; do @@ -791,13 +768,6 @@ jobs: docker buildx imagetools create --prefer-index=false \ -t "${STACK_IMAGE}:sha-${GITHUB_SHA:0:7}" "${STACK_IMAGE}@${DIGEST}" - - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac - - - name: sign the image - env: - DIGEST: ${{ steps.push.outputs.digest }} - run: cosign sign --yes "${STACK_IMAGE}@${DIGEST}" - - name: report what to pin env: DIGEST: ${{ steps.push.outputs.digest }} @@ -838,7 +808,6 @@ jobs: permissions: contents: read packages: write - id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -890,7 +859,7 @@ jobs: --set "*.args.POD_BASE_IMAGE=${POD_BASE_IMAGE}" --set "*.args.VOGT_SOURCE_REF=${VOGT_SOURCE_REF}" --set "*.args.VOGT_SOURCE_SHA=${VOGT_SOURCE_SHA}" - --set "*.args.VOGT_PRODUCT_VERSION=0.7.17" + --set "*.args.VOGT_PRODUCT_VERSION=0.7.22" ) # metadata-action's labels are newline-separated key=value. while IFS= read -r pair || [ -n "$pair" ]; do @@ -938,13 +907,6 @@ jobs: -t "${STACK_WORKBENCH_IMAGE}:sha-${GITHUB_SHA:0:7}" \ "${STACK_WORKBENCH_IMAGE}@${DIGEST}" - - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac - - - name: sign the image - env: - DIGEST: ${{ steps.push.outputs.digest }} - run: cosign sign --yes "${STACK_WORKBENCH_IMAGE}@${DIGEST}" - - name: report what to pin env: DIGEST: ${{ steps.push.outputs.digest }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 717c1697f..4c16aaedb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,11 @@ on: push: branches: [main] pull_request: + # The merge queue (WI-1886). A queued entry is the exact tree `main` will + # become, so it is classified as a change to everything: pull requests get + # the fast, path-scoped subset below and the queue is where the full suite + # runs before anything lands. See docs/ci.md. + merge_group: # Manual runs. workflow_dispatch: @@ -39,7 +44,10 @@ concurrency: # per-commit in the same change. # # The ref stays in the key: a PR groups by its own `refs/pull/N/merge` - # ref, so PRs never evict each other or a branch. + # ref, so PRs never evict each other or a branch, and a force-push cancels + # the superseded PR run. A merge-queue entry runs on its own + # `gh-readonly-queue/main/pr-N-` ref, so queue entries never cancel + # each other either; the queue itself removes an entry it no longer needs. group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -64,6 +72,9 @@ jobs: outputs: python: ${{ steps.classify.outputs.python }} engine: ${{ steps.classify.outputs.engine }} + rust: ${{ steps.classify.outputs.rust }} + web: ${{ steps.classify.outputs.web }} + full: ${{ steps.classify.outputs.full }} voice: ${{ steps.classify.outputs.voice }} mobile: ${{ steps.classify.outputs.mobile }} any: ${{ steps.classify.outputs.any }} @@ -86,6 +97,8 @@ jobs: { echo "python=true" echo "engine=true" + echo "rust=true" + echo "web=true" echo "voice=true" echo "mobile=true" echo "any=true" @@ -93,18 +106,22 @@ jobs: } if [ "$EVENT_NAME" != "pull_request" ]; then - # A push checks everything, always. The concurrency group is + # `full` widens the jobs that narrow themselves on a PR (the + # Python matrix, the core-alone job). Only a PR is ever partial. + echo "full=true" >> "$GITHUB_OUTPUT" + # A push or merge-queue run checks everything, always. The concurrency group is # newest-wins per ref, so a superseded push run is cancelled — # which is only safe because the replacing run re-checks the # whole tree. Classifying a push by `before..sha` under that # policy would mean a cancelled run's files are checked by # nothing, ever; the two escapes that found that hole are # recorded in the concurrency comment at the top of this file. - echo "push: newest-wins concurrency, so every push checks everything" + echo "$EVENT_NAME: every push and merge-queue entry checks everything" everything exit 0 fi + echo "full=false" >> "$GITHUB_OUTPUT" git fetch --no-tags origin "$BASE_REF" base="$(git merge-base "origin/$BASE_REF" HEAD)" range="$base..HEAD" @@ -127,7 +144,8 @@ jobs: fi python=false - engine=false + rust=false + web=false voice=false mobile=false @@ -141,12 +159,25 @@ jobs: # rust-embed, so the PWA is a *build input* to the Rust crate rather # than a neighbour of it: a web-only change changes what the engine # serves, and is checked by rebuilding the engine. - matches '^(engine/|web/)' && engine=true + # + # On a PR the engine job still splits in two halves so a PR pays + # only for the half it touched: `rust` (fmt, clippy, nextest, audit) + # and `web` (typecheck, vitest, demo Playwright, bundle budget). A + # web-only PR does not rebuild the crate against its new bundle; + # that embed check runs in the merge queue, where both are true. + matches '^engine/' && rust=true + matches '^web/' && web=true # The engine compiles `docs/ENGINE.md` in with `include_str!` (its # voice-policy test reads the capability matrix), so the document is # an engine input too. - matches '^docs/ENGINE\.md$' && engine=true - # The PWA contract check reads the Rust registry golden. + matches '^docs/ENGINE\.md$' && rust=true + engine=false + if [ "$rust" = true ] || [ "$web" = true ]; then + engine=true + fi + # The PWA contract check reads the Rust registry golden. This has to + # follow the engine=false reset above, or a registry-only change is + # classified as no engine work. matches '^tests/parity/golden/registry\.json$' && engine=true matches '^voice/' && voice=true # The shell wraps a remote URL, not the PWA bundle — `webDir` is @@ -157,6 +188,8 @@ jobs: { echo "python=$python" echo "engine=$engine" + echo "rust=$rust" + echo "web=$web" echo "voice=$voice" echo "mobile=$mobile" } >> "$GITHUB_OUTPUT" @@ -178,8 +211,9 @@ jobs: strategy: fail-fast: false matrix: - # The declared floor and a current release. - python-version: ["3.11", "3.13"] + # The declared floor and a current release. A PR runs the current + # release only; the floor runs in the merge queue and on `main`. + python-version: ${{ fromJSON(needs.changes.outputs.full == 'true' && '["3.11", "3.13"]' || '["3.13"]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -200,7 +234,7 @@ jobs: - run: uv sync --locked - name: canonical product version is aligned - run: cargo run --quiet --manifest-path engine/core/Cargo.toml --bin vogt-core -- check-product-version 0.7.17 + run: cargo run --quiet --manifest-path engine/core/Cargo.toml --bin vogt-core -- check-product-version 0.7.22 - name: ruff (lint) run: uv run --no-sync ruff check . @@ -233,7 +267,9 @@ jobs: core: name: the core alone, with no engine present needs: changes - if: needs.changes.outputs.any == 'true' + # The job deletes engine/, web/ and mobile/ before it tests, so on a PR + # only a Python change can change its result; the queue runs it always. + if: needs.changes.outputs.python == 'true' runs-on: [self-hosted] timeout-minutes: 20 env: @@ -264,12 +300,18 @@ jobs: name: fmt, clippy, test (engine) and the PWA it embeds needs: changes if: needs.changes.outputs.engine == 'true' - runs-on: [self-hosted] + # `tailnet` is carried by the Node B runners (node-b-gha-public-*) and + # not by the GCP rust-heavy pool, whose image has no ripgrep and whose + # `runner` user has no passwordless sudo — so the install step below + # fails there and the `/api/search` tests go red (WI-1887). Widen this + # back to the whole pool once that image is rebaked. + runs-on: [self-hosted, tailnet] timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + if: needs.changes.outputs.rust == 'true' with: components: rustfmt, clippy @@ -279,12 +321,14 @@ jobs: # and a pull request's cache is never reused by another ref, so only # main writes it back. Every ref still restores the cache main saved. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 + if: needs.changes.outputs.rust == 'true' with: workspaces: engine save-if: ${{ github.ref == 'refs/heads/main' }} # First: it needs nothing built, so it is the cheapest failure to reach. - name: cargo fmt + if: needs.changes.outputs.rust == 'true' working-directory: engine run: cargo fmt --all -- --check @@ -298,7 +342,7 @@ jobs: # one; this reads `cargo metadata` and fails when it does. - name: declared rust-version covers the lockfile # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' run: uv run --no-project python scripts/check_rust_version.py engine # cargo-audit is a fixed version, and compiling it from crates.io is @@ -312,7 +356,7 @@ jobs: - name: restore the cargo-audit binary id: cargo-audit # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /root/.cargo/bin/cargo-audit @@ -320,12 +364,12 @@ jobs: - name: install cargo-audit # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() && steps.cargo-audit.outputs.cache-hit != 'true' + if: always() && !cancelled() && steps.cargo-audit.outputs.cache-hit != 'true' && needs.changes.outputs.rust == 'true' run: cargo install --locked cargo-audit --version 0.22.2 --force - name: audit Rust dependencies # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' working-directory: engine run: | set -euo pipefail @@ -358,22 +402,31 @@ jobs: # RUSTSEC-2025-0134: rustls-pemfile is unmaintained and has no # successor release. It arrives through rustls-native-certs and # axum-server, which still depend on it, so dropping our direct use - # does not remove it from the lock. Both ignores stay explicit while - # the audit denies future vulnerabilities, unsound advisories, and yanks. + # does not remove it from the lock. + # + # RUSTSEC-2026-0387 (crypto-bigint 0.5.5 timing variability) has no + # reachable fix: the latest jsonwebtoken and web-push both pin + # elliptic-curve 0.13, which pins crypto-bigint 0.5. Unlike the rsa + # one, this path is used (VAPID ES256 signing, JWTs); the risk is a + # remote timing side channel. Accepted by the operator 2026-10-24; + # WI-2373 removes it when those crates reach elliptic-curve 0.14. + # These ignores stay explicit while the audit denies future + # vulnerabilities, unsound advisories, and yanks. cargo audit --no-fetch --db "$advisory_db" \ --deny warnings \ --ignore RUSTSEC-2023-0071 \ - --ignore RUSTSEC-2025-0134 + --ignore RUSTSEC-2025-0134 \ + --ignore RUSTSEC-2026-0387 - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' with: version: 11 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' with: node-version: "22" # The pnpm store, keyed on web/pnpm-lock.yaml, so a warm run does @@ -383,13 +436,13 @@ jobs: - name: install the PWA's dependencies # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web run: pnpm install --frozen-lockfile - name: audit the PWA's dependencies # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web # Fatal on advisories, resilient to registry timeouts. run: bash ../scripts/pnpm-audit.sh @@ -399,7 +452,7 @@ jobs: # bundle, which is how the engine's pipeline reported it. - name: typecheck the PWA # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web run: pnpm typecheck @@ -427,19 +480,19 @@ jobs: # failure to reach than a bundle, and these need no `dist`. - name: test the PWA # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web run: pnpm test - name: install the demo browser # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web run: pnpm exec playwright install --with-deps chromium - name: demo routes, phone composition, split layouts, and screenshots # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web env: CI: "true" @@ -457,7 +510,7 @@ jobs: # disagreement can be read, and the baseline re-cut from CI's own # rendering when the change is the intended one. - name: keep the demo browser evidence when a screenshot disagrees - if: failure() + if: failure() && needs.changes.outputs.web == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 with: name: demo-playwright-results @@ -472,7 +525,7 @@ jobs: # check the engine against assets it will never serve. - name: build the PWA # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' working-directory: web env: VOGT_SOURCE_REF: ${{ github.ref_name }} @@ -481,7 +534,7 @@ jobs: - name: normal and demo packaging share one PWA artifact # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' run: | set -euo pipefail cp -a web/dist web/dist-normal @@ -506,7 +559,7 @@ jobs: # environment synced into a job that is otherwise Rust and Node. - name: the first screen stays inside its budget # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.web == 'true' run: uv run --no-project python scripts/check_bundle.py - name: web/dist exists even when the PWA was not built @@ -530,7 +583,7 @@ jobs: - name: ripgrep and git are present # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' run: | set -eu # `server/src/files.rs` shells out to `rg` and `server/src/git.rs` @@ -543,10 +596,11 @@ jobs: command -v rg >/dev/null || missing="$missing ripgrep" if [ -n "$missing" ]; then echo "installing:$missing" - # The runner executes as root and has no sudoers entry, so - # `sudo apt-get` fails with "root is not in the sudoers file" — - # which is what the first run on this branch did. Elevate only - # when there is something to elevate from. + # Runners differ: some execute as root with no sudoers entry + # (where `sudo apt-get` fails with "root is not in the sudoers + # file"), others as an unprivileged user that may or may not + # have passwordless sudo. Use apt-get directly as root, sudo + # otherwise, and fail clearly when neither is available. if [ "$(id -u)" = "0" ]; then apt-get update apt-get install -y --no-install-recommends $missing @@ -567,7 +621,7 @@ jobs: # that does not fail is a lint nobody reads. - name: cargo clippy # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' working-directory: engine run: cargo clippy --locked --workspace --all-targets -- -D warnings @@ -577,7 +631,7 @@ jobs: # (WI-1312). `--no-fail-fast` lists every failing test in one round. - name: cargo nextest # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' working-directory: engine run: | set -euo pipefail @@ -589,7 +643,7 @@ jobs: # nextest does not run doctests, so they keep their own step. - name: cargo test (doctests) # Runs after a failed step too, so one round reports every failure. - if: always() && !cancelled() + if: always() && !cancelled() && needs.changes.outputs.rust == 'true' working-directory: engine run: cargo test --locked --workspace --doc diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3700acca9..627fad679 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,26 +1,67 @@ name: codeql # Repository-owned code scanning for all three implementation languages. -# This starts as an independent check: once the self-hosted runner has -# produced a receipt for every matrix entry it can be added to branch -# protection without turning runner/tool-download uncertainty into a merge -# outage. +# Operator decision: CodeQL runs only when a release build is cut. A beta or +# prod tag push (the same trigger release.yml and release-mobile.yml use) +# starts it, and so does a manual dispatch. The dev deploy lane dispatches it +# with `ref` set to the commit it deploys. It does not run on pull_request, +# merge_group, push to main, or a schedule. The job names are unchanged. +# See docs/ci.md, "CodeQL". on: push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: "23 3 * * 1" + tags: + - "v*" workflow_dispatch: + inputs: + ref: + description: "Commit SHA to analyse (default: the dispatched ref)" + required: false + type: string permissions: contents: read jobs: + changes: + name: classify languages + runs-on: [self-hosted] + timeout-minutes: 10 + outputs: + python: ${{ steps.classify.outputs.python }} + javascript-typescript: ${{ steps.classify.outputs.javascript-typescript }} + rust: ${{ steps.classify.outputs.rust }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + ref: ${{ inputs.ref }} + + - id: classify + env: + EVENT_NAME: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + echo "$EVENT_NAME: analysing every language" + printf 'python=true\njavascript-typescript=true\nrust=true\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + git fetch --no-tags origin "$BASE_REF" + files="$(git diff --name-only "$(git merge-base "origin/$BASE_REF" HEAD)..HEAD")" + matches() { printf '%s\n' "$files" | grep -Eq "$1"; } + # A change to the scanning setup itself re-analyses everything. + if matches '^\.github/(workflows/codeql\.yml|codeql/)'; then + printf 'python=true\njavascript-typescript=true\nrust=true\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + py=false; js=false; rs=false + matches '\.py$|^(pyproject\.toml|uv\.lock)$' && py=true + matches '\.(js|jsx|mjs|cjs|ts|tsx)$|(^|/)package\.json$|pnpm-lock\.yaml$' && js=true + matches '\.rs$|(^|/)Cargo\.(toml|lock)$' && rs=true + printf 'python=%s\njavascript-typescript=%s\nrust=%s\n' "$py" "$js" "$rs" >> "$GITHUB_OUTPUT" + analyze: name: analyze (${{ matrix.language }}) # `bigmem` is node-b-gha-public-bigmem (ops personal/github-runner, 32 G @@ -31,6 +72,7 @@ jobs: # on this runner autodetect has room, and a pin only slows the analysis. # A label no online runner advertises queues forever, so this runner must # stay up; there is one bigmem slot, so the languages run one after another. + needs: changes runs-on: [self-hosted, bigmem] timeout-minutes: 45 concurrency: @@ -50,13 +92,22 @@ jobs: - javascript-typescript - rust steps: + # Every release-triggered or dispatched run analyses every language, so + # the change classifier above always reports true for these events. + - name: nothing in this language changed + if: needs.changes.outputs[matrix.language] != 'true' + run: echo "no ${{ matrix.language }} change on this pull request; the merge queue analyses it" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: needs.changes.outputs[matrix.language] == 'true' + with: + ref: ${{ inputs.ref }} # The Actions runtime bundles Node for action execution, but the # JavaScript/TypeScript extractor starts `node` from PATH. The # self-hosted base runner deliberately has no language runtimes. - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 - if: matrix.language == 'javascript-typescript' + if: matrix.language == 'javascript-typescript' && needs.changes.outputs[matrix.language] == 'true' with: node-version: "22" @@ -65,6 +116,7 @@ jobs: # from the authoritative build/test jobs and avoids rebuilding the PWA # or engine only to extract source a second time. - uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d + if: needs.changes.outputs[matrix.language] == 'true' with: languages: ${{ matrix.language }} build-mode: none @@ -73,5 +125,6 @@ jobs: config-file: ./.github/codeql/codeql-config.yml - uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d + if: needs.changes.outputs[matrix.language] == 'true' with: category: /language:${{ matrix.language }} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index e2f4f8d13..ea489b645 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,6 +17,17 @@ name: dependency review on: pull_request: + # `dependency review` is required on `main`, so a merge-queue entry needs + # the context too or it waits until the queue times it out. The review + # itself compares a PR's base and head, which a `merge_group` event does + # not carry; the dependencies an entry adds were reviewed on its PR, and the + # queue only rebases them, so the queue run posts the context and skips the + # review (WI-1886). + merge_group: + +concurrency: + group: dependency-review-${{ github.ref }} + cancel-in-progress: true permissions: contents: read @@ -30,8 +41,14 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: github.event_name == 'pull_request' + + - name: reviewed on the pull request + if: github.event_name == 'merge_group' + run: echo "dependencies were reviewed on the pull request; the queue rebases them unchanged" - name: review changed dependencies + if: github.event_name == 'pull_request' uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 with: # A real gate, not a rubber stamp, but scoped to what a PR introduces: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 343f62e64..5d7e31375 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,13 +1,15 @@ name: release # Tag-triggered only: a push to main can never publish a release. And -# publishing is not deploying: this workflow promotes signed images; a +# publishing is not deploying: this workflow promotes built images; a # deployment moves only when its operator pins the published digest -# (docs/DEPLOYMENT.md). +# (docs/DEPLOYMENT.md). Nothing here signs: image signing was dropped +# (operator decision, 2026-10-20), so a digest is trusted by its registry +# name and attestations, not by a signature. # # ── Build once, promote by digest ───────────────────────────────────────── # -# A release does NOT build images. `build.yml` builds, smoke-tests, signs and +# A release does NOT build images. `build.yml` builds, smoke-tests and # publishes `vogt`, `vogt-stack`, `vogt-stack-workbench` and `vogt-voice` # once, for the commit on # `main`, tagged `sha-<7-char commit>`, with the product version already baked @@ -17,18 +19,15 @@ name: release # 1. prove the tag is on main and names the version the source carries; # 2. prove `build.yml` ran for this commit on main and its four image jobs # succeeded, resolve each `sha-` tag to its digest, and verify -# the digest carries build.yml's own signature for this exact commit plus -# its SBOM and provenance attestations — anything missing FAILS the -# release; nothing is ever rebuilt as a fallback; +# the digest carries its SBOM and provenance attestations — anything +# missing FAILS the release; nothing is ever rebuilt as a fallback; # 3. retag those digests `X.Y.Z`, `X.Y` and `latest` with # `docker buildx imagetools create --prefer-index=false` (a registry-side # copy of the original manifest bytes: no pull, no rebuild, no new index, # so the digest is unchanged — and every new tag is asserted to resolve # to it); -# 4. sign the same digests keylessly under THIS workflow's identity -# (`release.yml@refs/tags/v*`), which is what deployments verify; -# 5. build and sign the Android shell (the app is not a container image, so -# it is still built at tag time) and write the GitHub Release with a +# 4. build the Android shell (the app is not a container image, so it is +# still built at tag time) and write the GitHub Release with a # machine-readable `vogt-release-manifest.json` naming the source sha and # every digest, so a deployment can match it against its dev receipt. # @@ -48,7 +47,7 @@ permissions: env: # The core alone — the verifiable build input of the product (the stack # image copies `/opt/vogt` out of it by digest), not the product. Promoted - # and release-signed with the pair so the manifest records all four. + # and promoted with the pair so the manifest records all four. # Nothing tells an operator to deploy it (DEPLOYMENT.md). IMAGE: ghcr.io/thedancingdeveloper-org/vogt # The product: core plus session engine, PWA and agent CLIs in one @@ -205,10 +204,6 @@ jobs: permissions: contents: read packages: write - # Keyless signing: the workflow's own OIDC identity is the signer, so - # there is no key to store or rotate and the signature binds to this - # repository, this workflow and this tag. - id-token: write outputs: core: ${{ steps.resolve.outputs.core }} stack: ${{ steps.resolve.outputs.stack }} @@ -222,8 +217,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac - - id: tags name: the semver tags this release moves env: @@ -244,17 +237,15 @@ jobs: echo "will tag: ${tags}" - id: resolve - name: resolve and verify build.yml's digests for this commit + name: resolve and check build.yml's digests for this commit env: TAG_SHA: ${{ github.sha }} run: | set -euo pipefail build_tag="${BUILD_TAG_PREFIX}${TAG_SHA:0:${BUILD_TAG_SHA_LENGTH}}" - # Only build.yml, on main, at exactly this commit, may have signed - # what is promoted. The workflow-sha extension in the Fulcio - # certificate is what ties the digest to the full commit, not the - # short tag it was found by. - build_identity="https://github.com/${GITHUB_REPOSITORY}/.github/workflows/build.yml@refs/heads/main" + # The digest is found by build.yml's sha- tag for this commit, and the + # build-run gate above proved build.yml published it. There is no + # signature check: image signing was dropped. fail="" for pair in "core=${IMAGE}" "stack=${STACK_IMAGE}" "workbench=${STACK_WORKBENCH_IMAGE}" "voice=${VOICE_IMAGE}"; do name="${pair%%=*}"; repo="${pair#*=}" @@ -269,14 +260,6 @@ jobs: sha256:*) ;; *) echo "::error::${name}: could not read a digest for ${ref}"; fail=1; continue ;; esac - if ! cosign verify \ - --certificate-identity "$build_identity" \ - --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - --certificate-github-workflow-sha "$TAG_SHA" \ - "${repo}@${digest}" >/dev/null; then - echo "::error::${name}: ${repo}@${digest} carries no build.yml signature for ${TAG_SHA}; refusing to promote it." - fail=1; continue - fi # build.yml pushes with `provenance: true` and `sbom: true`, so the # attestations live in the image index beside the image and travel # with the digest. A release has always carried both; a digest @@ -288,7 +271,7 @@ jobs: esac done echo "${name}=${digest}" >> "$GITHUB_OUTPUT" - echo "${name}: ${ref} -> ${repo}@${digest} (build.yml-signed for ${TAG_SHA})" + echo "${name}: ${ref} -> ${repo}@${digest} (built by build.yml for ${TAG_SHA})" done if [ -n "$fail" ]; then echo "the build for ${TAG_SHA} cannot be promoted — see the errors above" >&2 @@ -328,23 +311,6 @@ jobs: echo "${repo}: ${TAGS} -> ${digest}" done - - name: sign the same digests as the release - env: - CORE_DIGEST: ${{ steps.resolve.outputs.core }} - STACK_DIGEST: ${{ steps.resolve.outputs.stack }} - WORKBENCH_DIGEST: ${{ steps.resolve.outputs.workbench }} - VOICE_DIGEST: ${{ steps.resolve.outputs.voice }} - run: | - set -euo pipefail - # The digest, never a tag: a tag can be moved after signing, and a - # signature over a movable reference proves nothing. This adds the - # release.yml@refs/tags/v* signature beside build.yml's; deployments - # verify the release identity (DEPLOYMENT.md §6). - cosign sign --yes "${IMAGE}@${CORE_DIGEST}" - cosign sign --yes "${STACK_IMAGE}@${STACK_DIGEST}" - cosign sign --yes "${STACK_WORKBENCH_IMAGE}@${WORKBENCH_DIGEST}" - cosign sign --yes "${VOICE_IMAGE}@${VOICE_DIGEST}" - - name: report what to pin env: TAGS: ${{ steps.tags.outputs.tags }} @@ -357,7 +323,7 @@ jobs: { echo "## Promoted (built once by build.yml, not rebuilt)" echo - echo "Built by ${BUILD_RUN}; tagged \`${TAGS}\` and release-signed:" + echo "Built by ${BUILD_RUN}; tagged \`${TAGS}\`:" echo echo '```' echo "VOGT_STACK_IMAGE=${STACK_IMAGE}@${STACK_DIGEST}" @@ -613,7 +579,6 @@ jobs: --arg build_run_id "$BUILD_RUN_ID" \ --arg build_run_url "$BUILD_RUN_URL" \ --arg release_run_url "$RELEASE_RUN_URL" \ - --arg repository "$GITHUB_REPOSITORY" \ --arg guide "https://github.com/TheDancingDeveloper-org/vogt/blob/main/docs/DEPLOYMENT.md" \ '{ schema: $schema, @@ -641,12 +606,6 @@ jobs: release_run_url: $release_run_url }, provenance: { - cosign: "keyless signatures over the same digests from .github/workflows/release.yml (at release) and .github/workflows/build.yml (at build)", - cosign_identities: { - release: "https://github.com/\($repository)/.github/workflows/release.yml@refs/tags/\($tag)", - build: "https://github.com/\($repository)/.github/workflows/build.yml@refs/heads/main" - }, - cosign_issuer: "https://token.actions.githubusercontent.com", sbom: "BuildKit SBOM and provenance attestations recorded in each image index by build.yml, carried by the digest" }, deployment_guide: $guide, diff --git a/.github/workflows/runner-policy.yml b/.github/workflows/runner-policy.yml index c89685b4c..e9aa7bc2c 100644 --- a/.github/workflows/runner-policy.yml +++ b/.github/workflows/runner-policy.yml @@ -28,9 +28,19 @@ name: Runner policy on: pull_request: + # Required on `main`, so the merge queue must get a result too (WI-1886). + merge_group: push: branches: [main] +concurrency: + # The gate reads the whole tree, so a newer run covers a superseded one. + group: runner-policy-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + jobs: runner-policy: runs-on: [self-hosted] diff --git a/AGENTS.md b/AGENTS.md index 488fb5e50..3e298c249 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -347,6 +347,11 @@ does not need an APK rebuild. defaults. Exposure and identity values such as a public URL or non-loopback bind address must be stated, not guessed. Read `docs/DEPLOYMENT.md` before changing Compose defaults. +- Landing is through the merge queue (`docs/ci.md`): push a branch and open + a PR; the PR runs only fast, path-scoped checks, so don't expect full CI on + it and don't re-run the full suite by hand. `land-pr` enqueues the PR and + waits on the queue's `merge_group` runs, where the full suite runs. Rebase + only when the queue reports a conflict. - CI intentionally runs on self-hosted runners. A queued job is a runner capacity/availability question, never a reason to change `runs-on`. Fork limitations and local equivalents are documented in diff --git a/README.md b/README.md index 2bacec0dd..9fdc569cd 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ terminates TLS in front of it, and read [SECURITY.md](SECURITY.md) first. - [Agent guide](docs/AGENT_GUIDE.md) — for an agent working through Vogt. - [Configuration reference](docs/CONFIG.md) — every setting, generated from the schema. +- [GitHub App](docs/GITHUB_APP.md) — connecting to GitHub as an App instead + of a personal token: permissions, installation, config. - [Customisation](docs/CUSTOMISATION.md) — the supported extension points. - [Engine](docs/ENGINE.md) — the session engine and web app: what it owns, its wire contract, the assistant, agent tasks. diff --git a/SECURITY.md b/SECURITY.md index d7b17464d..45cbf95ec 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -86,7 +86,10 @@ GitHub dependency graph and Dependabot security updates are enabled for every manifest Vogt ships (`.github/dependabot.yml`). Pull requests run the `dependency review` workflow, which blocks a change introducing a high or critical severity dependency advisory. CodeQL scans Python, TypeScript, -JavaScript, and Rust on pull requests, pushes to `main`, and weekly. +JavaScript, and Rust only when a release is cut: on a `v*` tag push (beta and +prod) or a manual `workflow_dispatch`. The dev deploy lane dispatches it with +`ref` set to the commit it deploys. It does not run on pull requests, the +merge queue, pushes to `main`, or a schedule. `.github/dependabot.yml` carries two deliberate exceptions. The CodeQL `init` and `analyze` actions are grouped into one update, because a PR that @@ -111,9 +114,6 @@ After enabling these workflows, configure repository Settings → Rules → Rule - `ci` - `dependency review` -- `analyze (python)` -- `analyze (javascript-typescript)` -- `analyze (rust)` - `runner-policy` Also enable **Dependency graph**, **Dependabot alerts**, and **Dependabot @@ -122,8 +122,8 @@ settings and therefore cannot be represented in tracked files; verify them after a repository transfer or visibility change. This is a public repository, and `pull_request`-triggered jobs run on the -project's self-hosted runner pool (`ci.yml`, `codeql.yml`, -`runner-policy.yml`, `docs.yml`, `mirror-base-images.yml`). Those jobs run +project's self-hosted runner pool (`ci.yml`, `runner-policy.yml`, `docs.yml`, +`mirror-base-images.yml`). Those jobs run `uv`/`pytest`, `pnpm install`, `cargo`, Playwright and `gradlew` over PR-controlled source — arbitrary code execution by design — on a pool that also runs the secret-bearing jobs. @@ -159,39 +159,34 @@ by where the pipeline runs. What the tree _does_ enforce: are gated on `github.event_name != 'pull_request'`, and no workflow uses `pull_request_target`. - Release images are not built at release time: `release.yml` promotes, by - digest, the images `build.yml` built on `main` (see *Release signing* - below). Pull-request jobs never publish or sign an image. + digest, the images `build.yml` built on `main` (see *Image trust* + below). Pull-request jobs never publish an image. If the approval setting is ever found disabled, treat every self-hosted runner as potentially compromised by fork-submitted code and rotate the credentials those runners can reach. -### Release signing: build once, promote by digest +### Image trust: build once, promote by digest -Every published image is signed keylessly with cosign, by digest, using the -workflow's own GitHub OIDC identity — there is no signing key to store or -rotate. A released image carries two signatures on the **same digest**: +No image is signed. Signing was dropped (operator decision, 2026-10-20): +`build.yml` and `release.yml` neither install cosign nor sign, and no job +requests an `id-token`. A digest is trusted by its registry name and its SBOM +and provenance attestations. -- `…/.github/workflows/build.yml@refs/heads/main`, made when `build.yml` built - and smoke-tested it for a commit on `main`; -- `…/.github/workflows/release.yml@refs/tags/vX.Y.Z`, made when the `v*` tag - promoted that digest. Deployments verify this identity - ([`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md) §6). - -`release.yml` never builds. Before it signs, it requires a `build.yml` run on -`main` for the tagged commit whose image jobs succeeded, and it verifies each -digest's `build.yml` signature with the certificate's workflow-sha pinned to -the tagged commit, so it can only promote what `build.yml` built from that -exact source. It retags by digest (`imagetools create --prefer-index=false`) -and asserts every semver tag resolves to the promoted digest. The SBOM and -provenance attestations are the ones BuildKit recorded in the image index at -build time; they travel with the digest. If any of this is missing the -release fails; there is no rebuild fallback. +`release.yml` never builds. It requires a `build.yml` run on `main` for the +tagged commit whose image jobs succeeded, finds each image by its `sha-` +commit tag, and checks that the digest carries the SBOM and provenance +attestations. It retags by digest (`imagetools create --prefer-index=false`) +and asserts every semver tag resolves to the promoted digest. The attestations +are the ones BuildKit recorded in the image index at build time; they travel +with the digest. If any of this is missing the release fails; there is no +rebuild fallback. A deployment pins the digest, which a tag cannot silently +move. **Accepted trade-off.** `build.yml` imports its BuildKit layer cache from the operator's plaintext, unauthenticated LAN registry (`VOGT_BUILDKIT_CACHE_REGISTRY`) to keep per-commit builds fast. Releases used to build cold precisely so that -a poisoned cache entry could not become a release-signed artefact; promoting +a poisoned cache entry could not become a release artefact; promoting the `main` build gives that up in exchange for shipping exactly the bytes the development lane ran. The cache is therefore part of the release trust boundary: it must stay reachable only from the runner hosts, and the diff --git a/config.example.toml b/config.example.toml index 3e87ab2ac..234d94000 100644 --- a/config.example.toml +++ b/config.example.toml @@ -76,10 +76,30 @@ # default policy: behaviour # ci_watch_notify_sessions = true +# When true, the session sweep treats an `AUTOPILOT: DONE` as done only if the session posted a `result` report (`session.report`) since its turn began; otherwise the row reads `waiting` with the reason `done without a result report`, so oversight asks for one instead of trusting the line. Off by default. +# default policy: behaviour +# autopilot_done_requires_result_report = false + # Deployment lanes `deployed_versions` reports on, each a table with `name`, `project` (a registered slug), `branch` (default `main`), and a source: `receipt_repo` + `receipt_path` (a JSON receipt a deploy pipeline commits, read through the configured forge) and/or `version_url` (a running instance's public JSON version endpoint, e.g. Vogt's `/api/config`). Empty means the `deploy-lanes` collector is not registered and deployed versions are reported as not configured. A receipt whose status says the deploy or its smoke test failed raises an Inbox alert. # default policy: behaviour # deploy_lanes = [{ name = "dev", project = "my-app", version_url = "https://dev.example.com/api/config" }] +# Provider accounts `usage.board` reports on, each a table with `name`, `agent` (`claude` or `codex`) and `dir` (that account's config directory). Read on request only: the newest limit and reset the account's own transcripts recorded. Empty, no account is reported. +# default policy: behaviour +# usage_accounts = [] + +# Base URL of a LiteLLM proxy whose keys' spend `usage.board` reads through `/key/info`. Unset, no key spend is read. +# default policy: behaviour +# usage_litellm_url = null + +# LiteLLM keys `usage.board` reports spend for: `name`, `key_env` (the environment variable holding the key) and an optional `daily_alert_usd`. +# default policy: behaviour +# usage_litellm_keys = [] + +# Default day-spend alert threshold, in USD, for every LiteLLM key without its own `daily_alert_usd`. Unset, no alert is raised. +# default policy: behaviour +# usage_daily_spend_alert_usd = null + # File types the marker collector reads. Configuration rather than a hard-coded list, because which extensions hold source is a workspace's business, not Vogt's. # default policy: behaviour # marker_file_extensions = [".py", ".rs", ".ts", ".tsx", ".js", ".jsx", ".go", ".java", ".rb", ".sh", ".sql", ".toml", ".yaml", ".yml", ".md"] @@ -100,6 +120,18 @@ # default policy: behaviour # github_token_file = null +# The ID or the client ID of a GitHub App for the forge adapter to authenticate as; GitHub accepts either as the JWT issuer. With `github_app_private_key_file` also set, GitHub reads and writes use short-lived installation tokens, which have their own rate limit, instead of `github_token_file`. The notifications collector stays on `github_token_file`, because notifications belong to a user and an App cannot read them. Not a secret, so plain config is fine. Setup, including the permissions the App needs, is in docs/GITHUB_APP.md. +# default policy: behaviour +# github_app_id = null + +# Pin every GitHub request to one installation of `github_app_id`. Unset, the installation is discovered per repository owner (`GET /repos/{owner}/{repo}/installation`, then `/orgs/{owner}/installation` or `/users/{owner}/installation`) and remembered, so one App installed on several accounts or organisations covers them all; an owner without the App is read with `github_token_file`. Publishing a new repository needs an owner to create it under: this installation's account or, unset, the App's only installation. An App with several installations must set this to publish. +# default policy: behaviour +# github_app_installation_id = null + +# Path to the GitHub App's PEM private key, which signs the short-lived JWT exchanged for installation tokens. A file for the same reason as `github_token_file`: the key never appears in the environment or a process listing. When this and `github_app_id` are set, the App takes precedence over `github_token_file` for every GitHub read and write except notifications. Unset, or unreadable, the personal token is used as before. +# default policy: behaviour +# github_app_private_key_file = null + # Path to a file holding a urlsafe-base64 Fernet key, used to encrypt per-actor forge Personal Access Tokens at rest. A linked PAT must be *recoverable* to call the upstream API on the actor's behalf, which is the deliberate opposite of the vogt-issued tokens in `0005_tokens` — those store only a hash because they never need recovery. Its absence is what switches account linking off: with no key there is nowhere safe to keep a token, so the feature is disabled rather than insecure, and the file-token fallback (`github_token_file`) keeps working for sweeps and unlinked actors. A file rather than an environment variable, so the key never appears in a process listing. # default policy: behaviour # forge_account_key_file = null @@ -180,6 +212,30 @@ # default policy: behaviour # sweep_interval_seconds = 900 +# How often `serve` evaluates armed watches (WI-2036) and wakes the sessions whose trigger fired. Zero disables the loop: watches are stored but nothing wakes. Vogt-side triggers (a work item's state, a session ending) fire within this interval; forge triggers read what the sweep observed, so they also wait on `sweep_interval_seconds`. +# default policy: behaviour +# watch_interval_seconds = 60 + +# Cache-aware compaction on a cold wake (WI-2037). When on, waking a hibernated session whose provider prompt cache has expired (hibernated longer than its `cold_wake_ttl_seconds`) and whose context is at least `cold_wake_threshold_tokens` starts a fresh conversation seeded with a structured summary — task, state, decisions, files, branch, next step — instead of resuming the transcript at full uncached price. Off by default; a session opts out with `cold_wake_opt_out` at start or hibernate. +# default policy: behaviour +# cold_wake_compaction = false + +# The context size, in tokens, at or above which a cold wake is compacted (WI-2037). A smaller conversation is cheap enough to re-read and is resumed as it is. +# default policy: behaviour +# cold_wake_threshold_tokens = 100000 + +# Prompt-cache lifetime per provider, in seconds (WI-2037): a session hibernated longer than this is treated as cold. Keys are `anthropic`, `openai`, `deepseek`, `google`, … A provider with no entry is never compacted, because its cache cannot be judged. Anthropic's 1-hour cache tier is `anthropic = 3600`. +# default policy: behaviour +# cold_wake_ttl_seconds = {"anthropic": 300, "deepseek": 300} + +# How often the forge posture collector re-reads a repository's update-automation settings (version-update config, vulnerability alerts, automated security fixes). Posture changes rarely and cost several API calls per repository, so the default is daily; sweeps in between skip it. Zero reads it on every sweep. +# default policy: behaviour +# forge_posture_interval_seconds = 86400 + +# When GitHub reports fewer core API requests than this left in the hour, forge reads stop until the limit resets instead of spending the rest. The limit is per GitHub user and shared with every other tool on that account. Zero never pauses early. +# default policy: behaviour +# github_rate_limit_floor = 500 + # How recently a subject must have been observed for a linked declared entity to count as `verified` rather than `stale`. Trust is computed from this, never hand-set. # default policy: behaviour # verify_horizon_hours = 24 diff --git a/deploy/.env.example b/deploy/.env.example index 08425be20..1901a9851 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -22,7 +22,7 @@ VOGT_BIND_IP=127.0.0.1 # The image to run. Pin a digest in anything you would call a deployment. # Ignored when you add deploy/vogt.build.yml, which builds from the checkout. -# VOGT_IMAGE=ghcr.io/thedancingdeveloper-org/vogt:0.7.17 +# VOGT_IMAGE=ghcr.io/thedancingdeveloper-org/vogt:0.7.22 # The uid the container runs as. The gid is always 0 — that is what makes the # data directory writable at any uid. Set this to whoever owns the files this diff --git a/deploy/stack.compose.yml b/deploy/stack.compose.yml index 26c1c4bbe..31945b3bb 100644 --- a/deploy/stack.compose.yml +++ b/deploy/stack.compose.yml @@ -43,7 +43,7 @@ services: # `prod-` tags in the same repository are the maintainer's own pods: # same CLIs, plus estate integrations that address one estate's # infrastructure and mean nothing outside it. See DEPLOYMENT.md. - image: ${VOGT_STACK_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt-stack:0.7.17} + image: ${VOGT_STACK_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt-stack:0.7.22} restart: unless-stopped # tini as PID 1 reaps the processes agent sessions leave behind. The @@ -264,7 +264,7 @@ services: # Digest-pin this for a real deployment, exactly as the stack # image above. Pin the release family (`X.Y.Z`, `latest`); the default is a # tag so the file is runnable as-is. - image: ${VOGT_VOICE_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt-voice:0.7.17} + image: ${VOGT_VOICE_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt-voice:0.7.22} restart: unless-stopped # It only reads its baked models and writes audio to tmpfs, so it can run # locked down even though the pod beside it cannot. diff --git a/deploy/vogt.compose.yml b/deploy/vogt.compose.yml index b87c0ef55..81ea5b531 100644 --- a/deploy/vogt.compose.yml +++ b/deploy/vogt.compose.yml @@ -29,7 +29,7 @@ services: # digest — publishing an image and moving a deployment are separate acts, # and a digest is the only form of "which image is this" a # rebuild cannot silently change. - image: ${VOGT_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt:0.7.17} + image: ${VOGT_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt:0.7.22} restart: unless-stopped # The image runs as any uid so long as the gid is 0: `/var/lib/vogt` is diff --git a/docs/AGENT_GUIDE.md b/docs/AGENT_GUIDE.md index 82ea1de14..ec71a18b4 100644 --- a/docs/AGENT_GUIDE.md +++ b/docs/AGENT_GUIDE.md @@ -202,10 +202,19 @@ The table is the deployment's, not yours. A person overrides fields with `preference_set` on `dispatch.profiles` (for example `{"implement": {"model": "grok-4.8"}}` changes only that one field). The same store holds `dispatch.max_workers_per_overseer` (`{"value": 5}`), -`dispatch.escalate_after_failed_reviews` (`{"value": 2}`) and -`dispatch.review_by_oversight_only` (`{"value": true}`). An agent cannot set +`dispatch.escalate_after_failed_reviews` (`{"value": 2}`), +`dispatch.review_by_oversight_only` (`{"value": true}`) and +`dispatch.budget_restarts_per_item` (`{"value": 1}`). An agent cannot set any `dispatch.*` key. +A session the engine stops at its budget always leaves a hand-off (WI-1833). +If the agent wrote none after the soft warning, the scheduled sweep writes a +minimal one from its git state (branch, head, uncommitted paths). Then, up to +`dispatch.budget_restarts_per_item` times per item and only when nothing else +has started on the item since, it starts a fresh worker with +`from_handoff`, using the stopped session's purpose. Each restart leaves a +`[vogt] budget restart:` note on the item. + Every session records `purpose`, `started_by_session` (the engine id of the session that started it) and `started_by_kind` (`person`, or the starter's engine role at that moment: `oversight` or `worker`). Vogt reads the last two @@ -477,6 +486,21 @@ Rules: they work, so `running` with a recent `last_output_at` is a long turn; a `last_output_at` many minutes old is worth a look. +#### Reporting progress and results + +Post a short typed report with `session_report` instead of leaving your +driver to read your transcript: `kind` (`progress`, `result`, `question` or +`blocker`), a one- or two-sentence `summary`, and what applies of `branch`, +`sha`, `pr`, `checks` (one per entry, with the outcome), `blockers` and +`next_step`, plus a `reason`. Omit `id` from inside a session Vogt started. +The report shows as `latest_report` on your `session_list` and +`session_sweep` row, raises an Inbox entry and is left as a note on your work +item. Post a `result` when the deliverable is done — a pushed branch, a +review verdict — before you print `AUTOPILOT: DONE`; a deployment may require +one before it takes DONE as done. Never paste a credential or raw command +output into a report: credential-shaped text is redacted before it is +stored, but that is a safety net, not a licence. + #### When you, or a session you drive, need a person If you cannot go on without a person — a decision, a credential, an action @@ -503,6 +527,15 @@ is yours to decide. `decision_withdraw` takes back one you no longer need. Nothing is applied when a decision is answered: act on the answer through the op that checks the action. +**Charters.** Standing instructions that apply to every session on a scope +(commit identity, forbidden names, the delivery footer) belong in a charter, +not in each brief: `charter_set` with `scope` (`workspace`, `project`, +`initiative`), `ref` (the slug) and `body`; The workspace charter is a person's to set: an agent gets +`person_required`. `charter_get` reads it, with +`history: true` for earlier versions. Every brief on that scope carries them +under `## Charters`, workspace first, then project, then initiative, then the +brief. A later one refines an earlier one. + **Autopilot.** `session_start` with `autopilot: true` adds an Autopilot section to the brief: when the agent's next step needs no person, it carries on in the same turn instead of ending it to announce the next item, and @@ -516,6 +549,18 @@ cap (`autopilot` and `autopilot_nudges` on `session_list`). A driver need not re-prompt an autopilot session — wait for it to be blocked or done. The flag is recorded on the start's audit row. +**Done claims are checked.** A `done` row on `session_sweep` for a session +bound to a work item carries `claim_check`: the worktree's branch and HEAD, +whether HEAD is pushed, how many commits it gained since the session started, +and the observed pull requests the reply names or that come from that branch +(state, merge time, CI). `mismatches` names each disagreement in words, for +example a claimed PR that merged before the session started, or no new +commits. The row's `attention_reason` repeats them, so the digest shows them. +When a session moves its own item to `review` or `done`, `work_transition` +returns the same check and records any mismatch as a note on the item. The +check never refuses anything. Facts it could not read are listed under +`unchecked`, so an empty `mismatches` does not mean the claim was verified. + **Klaudia owns its own loop.** `session_start(template="klaudia")` runs Klaudia in its TUI like the other agents (model, resume, permission posture and the brief work the same way). Klaudia iterates by itself toward a goal @@ -540,7 +585,13 @@ the Inbox as "asking for approval". an agent's `session_answer` to one is refused with `403 person_required`, and so is any `session_input` while it shows — on a modal dialog every keystroke, `esc` included, is an answer. Nothing is typed (WI-983). If you -are an overseer and a worker is waiting on one, do not try to get round it: +are an oversight session (role `oversight`), you may answer a worker's +`permission` prompt with `session_answer` or `session_input` keys: read the +command first and answer as its supervisor. The engine checks your role and +audits each answer as you, with the command (WI-2028); `read-outside-cwd` +prompts stay a person's. Driven Codex workers launch with `-a never` by +default, so they rarely ask. Otherwise, as a non-oversight session, if a +worker is waiting on one, do not try to get round it: it is already in the Inbox as "asking for approval"; if you cannot go on until it is answered, report that with `session_report_blocked`, naming the session and the command. A person answers it from the Inbox or with @@ -623,6 +674,22 @@ Two things worth knowing about that edge: it is **observed, never typed in** — `depends_on`, an open `implemented_by` PR does not block the item from completing; it feeds the item's phase and its ranking, nothing more. +### Wait on CI and merges with a watch, not a poll + +Don't spend turns polling a PR or CI. Hibernate with a trigger and Vogt wakes +you with the outcome: + +```text +session_hibernate(id=, reason="wait for review and merge", + wake_on={kind: "pr", project: "vogt", number: 42}) +session_hibernate(..., wake_on={kind: "ci", project: "vogt", pr: 42, + checks: ["build", "test"]}) +``` + +`watch_create` arms the same trigger without sleeping (an overseer can name +the session to wake; `kind: "session"` waits for a worker to end), +`watch_list` shows what is armed and what fired, `watch_cancel` disarms one. + ### The git story that results From the branch tips and the PR edge, `work_get` derives a diff --git a/docs/API.md b/docs/API.md index 243ef0ca7..648eb366f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -205,7 +205,18 @@ again. Together they are the overseer's loop: wait, read, act. `session.report_blocked` / `session.report_unblocked` (`work.write`, audited) let an agent say it is blocked on a person; the report shows as `blocked` on `session.list` and `session.screen`, raises an Inbox -`session.blocked` entry and a push. `session.start` takes `autopilot` (see +`session.blocked` entry and a push. `session.report` (`work.write`, audited) +is a session's own short typed report — `kind` `progress`, `result`, +`question` or `blocker`, with `summary`, `branch`, `sha`, `pr`, `checks`, +`blockers` and `next_step` (WI-2039). The newest one rides on every +`session.list`/`session.sweep` row as `latest_report` and on +`session.digest`; in digest mode it replaces `last_reply_excerpt` with a +one-line `report`. It raises an Inbox `session.report.` entry (the +newest per session) and is left as a note on the work item: the session's +bound item, or the one `work_item` names. A session reports only for +itself. With `autopilot_done_requires_result_report` on (off by default), the +sweep reads an `AUTOPILOT: DONE` with no `result` report this turn as +`waiting`. `session.start` takes `autopilot` (see [`AGENT_GUIDE.md`](AGENT_GUIDE.md)). Rows of `session.list` and `session.sweep` carry `running`. It holds the @@ -288,7 +299,9 @@ token, which is bound to a person, so its agent counted as that person: it could grant `bypass`, and its writes were attributed to them (WI-926). A plain shell keeps the pod's token, because the person at it is a person. -The posture shows on session rows as `permission_mode`. See +The posture shows on session rows as `permission_mode`. A row with +`posture_lost_on_resume: true` was woken without the bypass its screen showed +before it hibernated (WI-2252); a person re-grants it in the terminal. See [`ENGINE.md`, "Permission posture"](ENGINE.md#permission-posture). `session.start` also takes `account`, the name of a Claude Code subscription @@ -350,7 +363,34 @@ is ready (`wake_timeout_s`, default 120) and then types; the result says "hibernated"` and the kept screen. `session.keep_awake` pins a session against the engine's idle policy. A session hibernated by hand, or one whose work was handed on, is not brought back when the engine restarts; one the -idle or shutdown policy hibernated still is. `session.set_role` (`work.write`, audited) +idle or shutdown policy hibernated still is. `session.hibernate` also takes `wake_on`, a watch trigger (below): the watch +is armed before the session sleeps, and cancelled if the hibernate fails. + +`watch.create` (`work.write`, audited) arms a **watch** (WI-2036): a stored +trigger and the session to wake — the caller's own unless `session` names +another. `trigger.kind` is `pr` (`project`, `number`; fires when the PR reads +one of `on`, default `merged` and `closed`), `ci` (`project`, one of `sha` +or `pr`, and `checks`, the required check names; fires when each has been +observed and concluded on that revision — Vogt does not read branch +protection, so the caller names the gating checks), `work_item` (fires when it +is in one of `states`, default a terminal state) or `session` (fires when +another session has exited, hibernated or stopped; `until: "done"` also when +it declared itself done). The core's watch loop (`watch_interval_seconds`, +default 60) evaluates active watches against what Vogt already holds — the +forge observations the sweep collected, the item, the engine's session list +— so a forge trigger fires within a sweep of the event. A fired watch is +delivered through `session.input`: a hibernated session is woken (never +forced) and a one-line `[vogt] watch fired: …` notice, with the watch's +`note`, is typed. A refused delivery is retried with backoff (1, 2, 4, 8 minutes; +each wake waits at most 30 s), five attempts in all, then the watch reads +`failed` with `last_error`. Watches live in the declared store, +so they survive a restart; past `ttl_seconds` (default 24 h, at most 14 d) +one reads `expired`. `watch.list` reads them by `state` or `session`, with +each `outcome` and `delivered`; `watch.cancel` disarms one (a person, its +owner, or the session it wakes). The merge queue is not observed, so a PR +leaving the queue without merging is not a trigger yet. + +`session.set_role` (`work.write`, audited) nominates a session as `oversight` — the one supervising the others — or makes it a `worker` again; `session.start` takes the same `role`. **It is a person's nomination**: an agent principal gets `403 role_refused`, because @@ -427,6 +467,18 @@ asking session may. `decision.list` (`read`) filters by `state` (`pending`, be answered and implies no answer. Nothing is applied when a decision is answered. The design is `design/agent-harness-services.md` §2.3. +`charter.set` (`work.write`, audited; the `workspace` scope is a person's only, `403 person_required` for any agent, because it reaches every session) sets the standing instructions of a +scope (WI-2038): `scope` is `workspace`, `project` or `initiative`, `ref` the +project or initiative slug (none for the workspace), `body` Markdown (≤ 8000; +empty clears), optional `expected_version` (0 for never set; a stale one is +refused `409 charter_version_conflict`). Each set appends the next version, +recording who set it and why. `charter.get` (`read`) returns `current` and, +with `history: true`, earlier versions newest first. Every session brief +carries the charters of its scope in a `## Charters` section, broadest first +(workspace, then project, then initiative, then the rest of the brief); a +cleared or absent charter adds nothing. The same section is sent to the engine +as `charter`, which appends it to a Codex session's `developer_instructions`. + Only a session whose conversation id the engine knows can hibernate: Claude Code started by Vogt, or any `resume`. A shell needs `allow_shell`, and wakes as a fresh shell. The engine side is @@ -516,10 +568,14 @@ docker exec \ person_required`), and `session.start` reads the newest person's write. `dispatch.profiles` is `{"": {field: value}}` and overrides only the fields it names (`template`, `model`, `effort`, `budget_turns`, - `budget_wall_clock_minutes`, `worktree`, `repo_map`, `autopilot`, - `read_only`). `dispatch.max_workers_per_overseer` (default 5), - `dispatch.escalate_after_failed_reviews` (2) and - `dispatch.review_by_oversight_only` (true) are each `{"value": …}`. Each + `budget_turns_by_model`, `budget_wall_clock_minutes`, `worktree`, + `repo_map`, `autopilot`, `read_only`). `budget_turns_by_model` sizes the + turn budget by the model the start runs (`implement` gives `haiku` 200). + `dispatch.max_workers_per_overseer` (default 5), + `dispatch.escalate_after_failed_reviews` (2), + `dispatch.review_by_oversight_only` (true) and + `dispatch.budget_restarts_per_item` (1; `0` turns automatic restarts off) + are each `{"value": …}`. Each key is validated against its own shape, and any other `dispatch.*` key is refused. - **`place.metrics`** — `inbox_active` is the badge: the count under the @@ -609,6 +665,14 @@ store. Without roots the collector is not registered, and both reads say so in `ci_watch_notify_sessions` is off, types a one-line notice plus Enter into each live session started for the item. Subscribe to the event feed to wake on it; nothing needs to poll a forge. +- **`usage.board`** (`GET /api/usage-board`, `vogt usage board`, MCP + `usage_board`; `read`) — each `usage_accounts` entry's reported limit + windows (Codex `primary`/`secondary` with `used_percent` and `resets_at`; + a Claude limit-hit with its reset), `limited`, and Claude `tokens_5h`; + each `usage_litellm_keys` key's `/key/info` spend (never the key), and + `alerts` for keys with a one-day budget window at or over their day + threshold. Read on request; nothing is stored. An unreadable source says + why in `detail`. - **`deployed.versions`** (`GET /api/deployed-versions`, `vogt deployed-versions`, MCP `deployed_versions`; `read`) — one row per lane in `deploy_lanes`: `name` (the configured lane name; also echoed as `lane`), @@ -705,11 +769,19 @@ fits in context. `landing` (`base_branch`, `branch_pattern` with `{n}` for the work item number, `worker_lands_by` of `push_branch` or `open_pr`, `merge_by` of `oversight` or `operator`, `merge_mode` of `fast_forward` or `rebase`, - `linear_history`, `deploy_by`), `review` (`template`, `model`, `effort`, + `linear_history`, `deploy_by`, `initiative_bases`, `long_lived_branches`), + `review` (`template`, `model`, `effort`, `never_models`, `budget_turns`) and `verification` (`required_for_done`, `lanes`). `branch_pattern` must compile as a regex once `{n}` is a number, and `merge_mode` of `merge` is refused; both refusals name the field. `project.get` returns the blocks as stored. Nothing enforces them yet. + `initiative_bases` maps an initiative id or slug to the base its items cut + from and land onto (the Rust port on `rust-core`), and a session start's + task that names a branch from `long_lived_branches` (or an initiative base) + as its base is taken at its word. The brief, the worktree's `base_ref` and + `session.start`'s `effective_policy` all carry the resolved base. An + oversight session's own `mechanical` start, on a project where oversight + merges, is not told "do not merge". ## Deploy diagnostics diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 84482cc2d..b1806b7b6 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -29,12 +29,20 @@ named by `VOGT_CONFIG_FILE`, then the defaults shown here. | `ci_alert_branches` | `VOGT_CI_ALERT_BRANCHES` | list of strings | `main`, `master`, `prod` | behaviour | | `ci_alert_tags` | `VOGT_CI_ALERT_TAGS` | list of strings | `v*` | behaviour | | `ci_watch_notify_sessions` | `VOGT_CI_WATCH_NOTIFY_SESSIONS` | boolean | `True` | behaviour | +| `autopilot_done_requires_result_report` | `VOGT_AUTOPILOT_DONE_REQUIRES_RESULT_REPORT` | boolean | `False` | behaviour | | `deploy_lanes` | `VOGT_DEPLOY_LANES` | list of `DeployLane` tables | *(empty)* | behaviour | +| `usage_accounts` | `VOGT_USAGE_ACCOUNTS` | list of `UsageAccount` tables | *(empty)* | behaviour | +| `usage_litellm_url` | `VOGT_USAGE_LITELLM_URL` | string, optional | *(no default — must be set)* | behaviour | +| `usage_litellm_keys` | `VOGT_USAGE_LITELLM_KEYS` | list of `UsageKey` tables | *(empty)* | behaviour | +| `usage_daily_spend_alert_usd` | `VOGT_USAGE_DAILY_SPEND_ALERT_USD` | number, optional | *(no default — must be set)* | behaviour | | `marker_file_extensions` | `VOGT_MARKER_FILE_EXTENSIONS` | list of strings | `.py`, `.rs`, `.ts`, `.tsx`, `.js`, `.jsx`, `.go`, `.java`, `.rb`, `.sh`, `.sql`, `.toml`, `.yaml`, `.yml`, `.md` | behaviour | | `branch_binding_patterns` | `VOGT_BRANCH_BINDING_PATTERNS` | list of strings | `(?i)\bwi-?(?P\d+)\b`, `(?i)\bgh-(?P\d+)\b` | behaviour | | `branch_binding_template` | `VOGT_BRANCH_BINDING_TEMPLATE` | string | `wi-{number}` | behaviour | | `retention_days` | `VOGT_RETENTION_DAYS` | integer | `180` | behaviour | | `github_token_file` | `VOGT_GITHUB_TOKEN_FILE` | path, optional | *(no default — must be set)* | behaviour | +| `github_app_id` | `VOGT_GITHUB_APP_ID` | string, optional | *(no default — must be set)* | behaviour | +| `github_app_installation_id` | `VOGT_GITHUB_APP_INSTALLATION_ID` | integer, optional | *(no default — must be set)* | behaviour | +| `github_app_private_key_file` | `VOGT_GITHUB_APP_PRIVATE_KEY_FILE` | path, optional | *(no default — must be set)* | behaviour | | `forge_account_key_file` | `VOGT_FORGE_ACCOUNT_KEY_FILE` | path, optional | *(no default — must be set)* | behaviour | | `forge_token_files` | `VOGT_FORGE_TOKEN_FILES` | map of string to path | *(empty)* | behaviour | | `agent_activity_roots` | `VOGT_AGENT_ACTIVITY_ROOTS` | map of string to path | *(empty)* | behaviour | @@ -55,6 +63,12 @@ named by `VOGT_CONFIG_FILE`, then the defaults shown here. | `install_bootstrap_enabled` | `VOGT_INSTALL_BOOTSTRAP_ENABLED` | boolean | `True` | behaviour | | `sqlite_synchronous` | `VOGT_SQLITE_SYNCHRONOUS` | one of `off`, `normal`, `full`, `extra` | `normal` | behaviour | | `sweep_interval_seconds` | `VOGT_SWEEP_INTERVAL_SECONDS` | integer | `900` | behaviour | +| `watch_interval_seconds` | `VOGT_WATCH_INTERVAL_SECONDS` | integer | `60` | behaviour | +| `cold_wake_compaction` | `VOGT_COLD_WAKE_COMPACTION` | boolean | `False` | behaviour | +| `cold_wake_threshold_tokens` | `VOGT_COLD_WAKE_THRESHOLD_TOKENS` | integer | `100000` | behaviour | +| `cold_wake_ttl_seconds` | `VOGT_COLD_WAKE_TTL_SECONDS` | map of string to integer | `{'anthropic': 300, 'deepseek': 300}` | behaviour | +| `forge_posture_interval_seconds` | `VOGT_FORGE_POSTURE_INTERVAL_SECONDS` | integer | `86400` | behaviour | +| `github_rate_limit_floor` | `VOGT_GITHUB_RATE_LIMIT_FLOOR` | integer | `500` | behaviour | | `verify_horizon_hours` | `VOGT_VERIFY_HORIZON_HOURS` | integer | `24` | behaviour | | `image_digest` | `VOGT_IMAGE_DIGEST` | string, optional | *(no default — must be set)* | behaviour | | `diagnostics_peer_url` | `VOGT_DIAGNOSTICS_PEER_URL` | string, optional | *(no default — must be set)* | exposure | @@ -134,10 +148,30 @@ Tags (glob patterns) whose failed CI runs raise an Inbox alert — the release w When CI finishes on a branch bound to a work item (`work.bind_branch`), type a one-line pass/fail notice into each live session started for that item, so an agent wakes on the result instead of polling. Sent once per run conclusion, by the sweep that observed it. The Inbox entry is raised either way; this only turns the session nudge off. +### `autopilot_done_requires_result_report` + +When true, the session sweep treats an `AUTOPILOT: DONE` as done only if the session posted a `result` report (`session.report`) since its turn began; otherwise the row reads `waiting` with the reason `done without a result report`, so oversight asks for one instead of trusting the line. Off by default. + ### `deploy_lanes` Deployment lanes `deployed_versions` reports on, each a table with `name`, `project` (a registered slug), `branch` (default `main`), and a source: `receipt_repo` + `receipt_path` (a JSON receipt a deploy pipeline commits, read through the configured forge) and/or `version_url` (a running instance's public JSON version endpoint, e.g. Vogt's `/api/config`). Empty means the `deploy-lanes` collector is not registered and deployed versions are reported as not configured. A receipt whose status says the deploy or its smoke test failed raises an Inbox alert. +### `usage_accounts` + +Provider accounts `usage.board` reports on, each a table with `name`, `agent` (`claude` or `codex`) and `dir` (that account's config directory). Read on request only: the newest limit and reset the account's own transcripts recorded. Empty, no account is reported. + +### `usage_litellm_url` + +Base URL of a LiteLLM proxy whose keys' spend `usage.board` reads through `/key/info`. Unset, no key spend is read. + +### `usage_litellm_keys` + +LiteLLM keys `usage.board` reports spend for: `name`, `key_env` (the environment variable holding the key) and an optional `daily_alert_usd`. + +### `usage_daily_spend_alert_usd` + +Default day-spend alert threshold, in USD, for every LiteLLM key without its own `daily_alert_usd`. Unset, no alert is raised. + ### `marker_file_extensions` File types the marker collector reads. Configuration rather than a hard-coded list, because which extensions hold source is a workspace's business, not Vogt's. @@ -158,6 +192,18 @@ How long observation *history* is kept. The newest observation per subject is ke Path to a file containing a GitHub token. Its absence is what switches the optional forge adapter off, so there is no default: not configured is the ordinary case, and it means forge subjects are 'not collected' rather than absent. A file rather than an environment variable or an argument, so the token never appears in a process listing. +### `github_app_id` + +The ID or the client ID of a GitHub App for the forge adapter to authenticate as; GitHub accepts either as the JWT issuer. With `github_app_private_key_file` also set, GitHub reads and writes use short-lived installation tokens, which have their own rate limit, instead of `github_token_file`. The notifications collector stays on `github_token_file`, because notifications belong to a user and an App cannot read them. Not a secret, so plain config is fine. Setup, including the permissions the App needs, is in docs/GITHUB_APP.md. + +### `github_app_installation_id` + +Pin every GitHub request to one installation of `github_app_id`. Unset, the installation is discovered per repository owner (`GET /repos/{owner}/{repo}/installation`, then `/orgs/{owner}/installation` or `/users/{owner}/installation`) and remembered, so one App installed on several accounts or organisations covers them all; an owner without the App is read with `github_token_file`. Publishing a new repository needs an owner to create it under: this installation's account or, unset, the App's only installation. An App with several installations must set this to publish. + +### `github_app_private_key_file` + +Path to the GitHub App's PEM private key, which signs the short-lived JWT exchanged for installation tokens. A file for the same reason as `github_token_file`: the key never appears in the environment or a process listing. When this and `github_app_id` are set, the App takes precedence over `github_token_file` for every GitHub read and write except notifications. Unset, or unreadable, the personal token is used as before. + ### `forge_account_key_file` Path to a file holding a urlsafe-base64 Fernet key, used to encrypt per-actor forge Personal Access Tokens at rest. A linked PAT must be *recoverable* to call the upstream API on the actor's behalf, which is the deliberate opposite of the vogt-issued tokens in `0005_tokens` — those store only a hash because they never need recovery. Its absence is what switches account linking off: with no key there is nowhere safe to keep a token, so the feature is disabled rather than insecure, and the file-token fallback (`github_token_file`) keeps working for sweeps and unlinked actors. A file rather than an environment variable, so the key never appears in a process listing. @@ -238,6 +284,30 @@ How hard SQLite works to survive a power cut. `normal` is the standard pairing f How often `serve` runs collectors in the background. Zero disables the schedule, leaving sweeps on-demand only. A default rather than a required value because an instance that never looks is the failure this product exists to prevent: stale evidence and no evidence are indistinguishable from the outside, so the safe default is to keep looking. +### `watch_interval_seconds` + +How often `serve` evaluates armed watches (WI-2036) and wakes the sessions whose trigger fired. Zero disables the loop: watches are stored but nothing wakes. Vogt-side triggers (a work item's state, a session ending) fire within this interval; forge triggers read what the sweep observed, so they also wait on `sweep_interval_seconds`. + +### `cold_wake_compaction` + +Cache-aware compaction on a cold wake (WI-2037). When on, waking a hibernated session whose provider prompt cache has expired (hibernated longer than its `cold_wake_ttl_seconds`) and whose context is at least `cold_wake_threshold_tokens` starts a fresh conversation seeded with a structured summary — task, state, decisions, files, branch, next step — instead of resuming the transcript at full uncached price. Off by default; a session opts out with `cold_wake_opt_out` at start or hibernate. + +### `cold_wake_threshold_tokens` + +The context size, in tokens, at or above which a cold wake is compacted (WI-2037). A smaller conversation is cheap enough to re-read and is resumed as it is. + +### `cold_wake_ttl_seconds` + +Prompt-cache lifetime per provider, in seconds (WI-2037): a session hibernated longer than this is treated as cold. Keys are `anthropic`, `openai`, `deepseek`, `google`, … A provider with no entry is never compacted, because its cache cannot be judged. Anthropic's 1-hour cache tier is `anthropic = 3600`. + +### `forge_posture_interval_seconds` + +How often the forge posture collector re-reads a repository's update-automation settings (version-update config, vulnerability alerts, automated security fixes). Posture changes rarely and cost several API calls per repository, so the default is daily; sweeps in between skip it. Zero reads it on every sweep. + +### `github_rate_limit_floor` + +When GitHub reports fewer core API requests than this left in the hour, forge reads stop until the limit resets instead of spending the rest. The limit is per GitHub user and shared with every other tool on that account. Zero never pauses early. + ### `verify_horizon_hours` How recently a subject must have been observed for a linked declared entity to count as `verified` rather than `stale`. Trust is computed from this, never hand-set. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 177736170..9ef9f7f9b 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -358,11 +358,13 @@ Actions expression in `runs-on`, and any third-party action not pinned to a full commit SHA. A queued job is a runner-capacity question; changing `runs-on` is not the fix and fails the gate. -[`ci.yml`](../.github/workflows/ci.yml) runs on every pull request and on -pushes to `main`. A `changes` job classifies the diff and runs only the halves -it touches; a change to anything shared (workflows, the Dockerfile, `deploy/`, -`.gitignore`) or any push runs everything. The halves are `build` (the Python -gates on 3.11 and 3.13, with the generated-config drift check and the +[`ci.yml`](../.github/workflows/ci.yml) runs on every pull request, in the +merge queue (`merge_group`) and on pushes to `main`. On a pull request a +`changes` job classifies the diff and runs only the halves it touches, with +the engine job split into its Rust and web halves; a change to anything shared +(workflows, the Dockerfile, `deploy/`, `.gitignore`) runs everything. The queue +and `main` always run everything. [`docs/ci.md`](ci.md) has the full table. +The halves are `build` (the Python gates on 3.13, plus 3.11 in the queue, with the generated-config drift check and the product-version check), `core` (the Python suite again with `engine/`, `web/` and `mobile/` deleted first), `engine` (Rust fmt, `cargo audit`, clippy and tests, with the PWA type-checked, tested, its demo Playwright project run, @@ -375,6 +377,14 @@ docs-only diff and fails whenever any half failed. **Required status checks on `main` are `ci` and `runner-policy`.** The `e2e` workflow (§5) is not required; it runs after a merge and nightly. +**Landing: the merge queue.** Push a branch and open a pull request. The PR +gets only the fast, path-scoped checks above, so do not expect, or re-run by +hand, the full suite on it. Land with `land-pr`, which enqueues the PR +(`gh pr merge --auto`) and follows the queue's `merge_group` runs. The full +suite runs there, against the exact tree `main` will become. A failed queue run +dequeues the PR: fix, push, re-run `land-pr`. Rebase only when the queue +reports a conflict, never just because `main` moved. + **Pull requests from forks.** A fork has no runner registered to it, so the workflows do not run on the fork's own Actions. A pull request from a fork runs against this repository's runners only after a maintainer approves the diff --git a/docs/CUSTOMISATION.md b/docs/CUSTOMISATION.md index a9b043121..9c78245c4 100644 --- a/docs/CUSTOMISATION.md +++ b/docs/CUSTOMISATION.md @@ -390,7 +390,7 @@ pattern: a Dockerfile a few lines long that starts `FROM` the published `vogt-stack` digest and adds what your sessions need. Keep it in your fork, build it in your CI, deploy it by digest. -The core image (`ghcr.io/thedancingdeveloper-org/vogt:0.7.17`) is a build +The core image (`ghcr.io/thedancingdeveloper-org/vogt:0.7.22`) is a build input to the stack image rather than something to deploy on its own. It is also built to be a base — `USER root`, install, `USER 1000:0`, keeping `ENTRYPOINT ["vogt"]` and the `root:0`-owned data directory — so extend it @@ -408,7 +408,7 @@ it adds, never in the mechanism. CI builds and boots the example with no private integrations, so the contract stays honest. **Derive from the published digest.** An extending image starts `FROM` the -signed release, by digest, not by tag: +release, by digest, not by tag: ```dockerfile FROM ghcr.io/thedancingdeveloper-org/vogt-stack@sha256: diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index be35ec11d..b79d04943 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -42,7 +42,7 @@ sidecar, released as a pair and run by one Compose file, on by default (`COMPOSE_PROFILES=voice`). The engine is not optional: it is the only way in. The core image, -`ghcr.io/thedancingdeveloper-org/vogt:0.7.17`, is also published at every +`ghcr.io/thedancingdeveloper-org/vogt:0.7.22`, is also published at every release because the stack image is built from it by digest and the release manifest records both — it is a build input, not a deployment target. `deploy/vogt.compose.yml` and `deploy/engine.overlay.yml` run a core and an @@ -136,8 +136,8 @@ Everything an operator chooses lives in `deploy/.env`, read by | `ENGINE_BIND` | no | `127.0.0.1` | Host interface the port is published on. Loopback until you mean to expose it. | | `ENGINE_PORT` | no | `8910` | Host port the container's 8910 is published on. | | `ENGINE_PUBLIC_URL` | no | — | The URL clients reach the stack at. Set it once there is a stable one (§4). | -| `VOGT_STACK_IMAGE` | no | `ghcr.io/thedancingdeveloper-org/vogt-stack:0.7.17` | The image to run. Pin a digest (§6). | -| `VOGT_VOICE_IMAGE` | no | `ghcr.io/thedancingdeveloper-org/vogt-voice:0.7.17` | The sidecar. Pin the same release as the stack. | +| `VOGT_STACK_IMAGE` | no | `ghcr.io/thedancingdeveloper-org/vogt-stack:0.7.22` | The image to run. Pin a digest (§6). | +| `VOGT_VOICE_IMAGE` | no | `ghcr.io/thedancingdeveloper-org/vogt-voice:0.7.22` | The sidecar. Pin the same release as the stack. | | `COMPOSE_PROFILES` | no | `voice` | Clear it to run without the sidecar; the voice controls stay present but inert. | | `VOGT_BOOTSTRAP_CORE_TOKEN_ACTOR` | no | `agent:engine` | Who the adopted stack secret acts as — and therefore whom a break-glass token's Vogt calls are attributed to. | | `VOGT_BOOTSTRAP_CORE_TOKEN_SCOPES` | no | `read,work.write,project.write` | What it may do. Everything in the pod can read the file, so this is the blast radius. | @@ -152,7 +152,11 @@ Everything an operator chooses lives in `deploy/.env`, read by | `ENGINE_AGENT_CODEX_POLICY` | no | the image file, else `/etc/codex/requirements.toml` | The Codex half. Codex itself reads `/etc/codex/requirements.toml` and no other path; the engine checks that file, and a named one too, is readable and wires the gitguard hook (`[[hooks.SessionStart]]` + `[[hooks.PreToolUse]]` running `/usr/local/bin/klaudia hook gitguard` with `failClosed = true`, plus the `[features] hooks = true` pin that stops a session switching hooks off) before a Codex launch. `/etc/codex/requirements.toml` and its directory must also be root-owned and not group- or world-writable, and `/usr/local/bin/klaudia` a root-owned regular file the same way; the hook always runs that image build, never a runtime `VOGT_KLAUDIA_VERSION` copy, so a gitguard fix ships only with an image rebuild (WI-1146). This is an accident guard, not a sandbox: the session user's passwordless sudo can defeat it. Empty checks `/etc/codex/requirements.toml`; the image's reference copy at `/usr/local/share/vogt/driven-session-codex-requirements.toml` without it is a refusal. `off` is the only way a deployment that has the file skips the check. A path that is missing, empty, unreadable or without that hook refuses the Codex launch (400) and does not fall through to bypass; Claude and opencode are not blocked. The shipped file wires the hook for the Codex fork (WI-1356) ([`ENGINE.md`, "Permission posture"](ENGINE.md#permission-posture)). | | `ENGINE_SESSION_RSS_WARN` | no | off | Flag any session whose process tree's resident memory reaches this (`8GiB`). Usage is shown either way; this only marks the heavy ones. | | `ENGINE_WORKTREE_GITHUB_TOKEN` | no | unset | GitHub token the engine uses when `session.start` fetches the worktree base from a GitHub `origin` (WI-1666). Set it to the same token sessions receive as `GH_TOKEN` (the manifest entry named by `ENGINE_AGENT_AUTH_GH_TOKEN_FROM`). A file is not used: the engine reads the variable and passes it to `GIT_ASKPASS` for that one fetch, and redacts it from errors. Unset, an `https://github.com` origin refuses the worktree start instead of failing as `Repository not found`. SSH GitHub origins and non-GitHub origins do not need it and keep the engine user's own git credentials. | -| `ENGINE_WORKTREE_CLEAN_ON_HIBERNATE` | no | off | When a session is hibernated or stopped and its directory is a linked git worktree, remove regenerable build output there (`target` beside a `Cargo.toml`, `node_modules`, `.venv`). A repo's main checkout is never touched, and a tracked file is never deleted ([`ENGINE.md`, "Worktree build output"](ENGINE.md#worktree-build-output)). | +| `ENGINE_WORKTREE_CLEAN_ON_HIBERNATE` | no | stop only | When a session is stopped or removed and its directory is a linked git worktree, remove regenerable build output there; on by default (WI-1750). `true` also cleans on hibernation (a hibernated session is woken to carry on, and every engine shutdown hibernates every live session, so this costs a full rebuild after each restart); `0` or `false` keeps build output. (`target` beside a `Cargo.toml`, `node_modules`, `.venv`). A repo's main checkout is never touched, and a tracked file is never deleted ([`ENGINE.md`, "Worktree build output"](ENGINE.md#worktree-build-output)). | +| `ENGINE_SCRATCH_PRUNE_GB` | no | `2` | With cleanup on, a hibernation removes Cargo `target` dirs from the session's Claude scratchpad and any top-level entry there over this many GiB; a stop or remove deletes the scratchpad ([`ENGINE.md`](ENGINE.md#session-scratch-background-processes-and-free-space), WI-1810). | +| `ENGINE_MIN_FREE_PERCENT` | no | `10` | Refuse new sessions (507) while the workspace or temp volume has less free space than this percent. `0` turns the guard off. | +| `ENGINE_SHARED_CARGO_TARGET_DIR` | no | `/claude-/cargo-target` | Shared Cargo target root for builds inside session scratch directories (one per repository, via a scratch-only `.cargo/config.toml`); `off` disables. Sessions' own worktrees are unaffected. | +| `ENGINE_SHARED_CARGO_TARGET_MAX_GB` | no | `40` | A repository's shared scratch target over this is removed when a session stops. | | `ENGINE_DISK_WARN_PERCENT` | no | `85` | Flag the workspace volume on `sweep`, `status` and `engine.status` once this percent of it is used. A whole percent from 1 to 100; anything else fails the boot. | | `ENGINE_ECONOMICS_TICK_SECS` | no | `15` | How often the engine reads each live session's transcript to meter its turns, context and compactions, reported on the session summary and as histograms on `GET /metrics` ([`ENGINE.md`, "Session economics"](ENGINE.md#session-economics)). `0` disables it. | | `ENGINE_HIBERNATE_IDLE_AFTER` | no | off | Hibernate an agent session quiet this long (`2h`, `30m`): its processes stop, it stays listed, and a wake resumes its conversation ([`ENGINE.md`, "Hibernation"](ENGINE.md#hibernation)). Sessions running a turn, at a permission dialog, blocked on a person, running a tool shell, pinned awake or on autopilot are exempt (autopilot only from this idle trigger). | @@ -610,28 +614,17 @@ a deployment are separate acts, and the digest line is what moves one. Resolve the digests of a release: ```console -docker buildx imagetools inspect ghcr.io/thedancingdeveloper-org/vogt-stack:0.7.17 \ +docker buildx imagetools inspect ghcr.io/thedancingdeveloper-org/vogt-stack:0.7.22 \ | grep -m 1 Digest -docker buildx imagetools inspect ghcr.io/thedancingdeveloper-org/vogt-voice:0.7.17 \ +docker buildx imagetools inspect ghcr.io/thedancingdeveloper-org/vogt-voice:0.7.22 \ | grep -m 1 Digest ``` -Verify the keyless signatures before starting Compose. Every release image is -signed by the release workflow's own OIDC identity, so the check constrains -both the workflow and the issuer, and performs an anonymous registry read. -(The same digest also carries the signature `build.yml` made when it built it -on `main` — a release promotes that build rather than rebuilding, §9 — but -the release identity is the one that says "this is version X".) - -```console -cosign verify \ - --certificate-identity-regexp '^https://github.com/TheDancingDeveloper-org/vogt/.github/workflows/release.yml@refs/tags/v[0-9].*$' \ - --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - ghcr.io/thedancingdeveloper-org/vogt-stack@sha256: -``` - -The same command applies to `vogt-voice@sha256:` unchanged. Then, in -`deploy/.env`: +Images are not signed (signing was dropped), so there is no signature to +verify. Pin the digest +itself, which a registry tag cannot silently move. Check that the digest +matches the release manifest (`vogt-release-manifest.json`) and the dev +receipt it was promoted from (§9). Then, in `deploy/.env`: ```dotenv VOGT_STACK_IMAGE=ghcr.io/thedancingdeveloper-org/vogt-stack@sha256: @@ -705,8 +698,8 @@ docker build -f engine/Dockerfile --target demo-runtime \ --build-arg VOGT_SOURCE_SHA="$(git rev-parse HEAD)" . ``` -The demo image is published by builds on `main` (§9), scanned and signed; -verify it with the `build.yml` workflow identity rather than the release one. +The demo image is published by builds on `main` (§9) and scanned; pin it by +the digest that `build.yml` reports. ## 9. Releases @@ -717,8 +710,8 @@ from it. A release publishes, for each of the core image (`vogt`), the stack - tags `X.Y.Z`, `X.Y` and `latest` — `latest` moves on a release and on nothing else (a pre-release tag such as `v1.0.0-rc.1` gets only its exact version); -- a keyless **cosign signature** over the digest, bound to - `release.yml@refs/tags/v*` (§6); +- no image signature (signing was dropped); the digest is the pin, and the + SBOM and provenance attestations are the record of what was built; - an **SBOM** and **provenance** attestation attached to the digest. ### Release process: build once, promote by digest @@ -733,8 +726,7 @@ existed: 2. **Build once.** `build.yml` builds the three images for that commit, runs each before pushing it — the stack must start both halves and both agent CLIs, the sidecar must synthesise and transcribe with its baked models — - then publishes them as `sha-<7-char commit>` with SBOM and provenance and - signs each digest under `build.yml@refs/heads/main`. + then publishes them as `sha-<7-char commit>` with SBOM and provenance. 3. **Validate.** A development deployment pins those digests and runs its smoke test against them. 4. **Tag** `vX.Y.Z` on that same commit. `release.yml` then: @@ -744,15 +736,12 @@ existed: - finds a `build.yml` run for the exact commit on `main` whose three image jobs succeeded (the demo image's job is not required), resolves each `sha-` tag to its digest, and verifies that digest carries - `build.yml`'s signature **for that commit** (the certificate's - workflow-sha) plus its SBOM and provenance attestations; + its SBOM and provenance attestations; - retags each digest `X.Y.Z`, `X.Y` and `latest` with `docker buildx imagetools create --prefer-index=false @` — a registry-side copy of the original manifest, no pull and no rebuild — and asserts every new tag resolves to exactly the source digest; - - signs those same digests with cosign under its own identity, so both a - build signature and a release signature sit on one digest; - - builds and signs the Android APK (the app is not a container image, so + - builds and signs the Android APK (Android app signing, not cosign; the app is not a container image, so it is still built at tag time; `release-mobile.yml` builds the Play AAB the same way) and writes a GitHub Release carrying the APK and `vogt-release-manifest.json`. @@ -772,12 +761,11 @@ refuse anything else. | `build.yml never ran for on main` | the tagged commit only touched docs (`build.yml` ignores `docs/**` and `*.md`), or a newer push cancelled its build before it started | tag the commit `build.yml` built (normally the bump), or dispatch `build.yml` on `main` while that commit is its head; then re-run the release | | `did not publish every image` | an image job of that `build.yml` run failed or was cancelled | `gh run rerun --failed`, then re-run the failed release jobs | | `:sha- is not in the registry` | the tag was never pushed or has been pruned | as above | -| `carries no build.yml signature for ` | the `sha-` tag now names a digest some other commit's build pushed (a 7-character collision), or the image is unsigned | investigate before releasing; never re-sign by hand | | `has no SBOM / Provenance attestation` | the digest was not pushed by `build.yml` | as above | | `: resolves to , not the promoted ` | the retag did not preserve the digest | do not deploy the tag; pin `@` from the manifest | The tag-time `release.yml` run is idempotent: re-running it re-resolves the -same digests, re-points the same tags and adds another release signature. +same digests and re-points the same tags. The shell is a remote WebView onto whichever deployed stack the user picks on-device at first launch, so a server or PWA release reaches installed @@ -792,7 +780,7 @@ deployment already running the `sha-` build of the tagged commit is already running the release; only its pin's spelling changes. Pushes to `main` are **builds, not releases**. `build.yml` publishes the same -three images tagged `sha-` (signed, with SBOM and provenance) and the +three images tagged `sha-` (with SBOM and provenance) and the demo image as `main` and `main-`. No semver tag is created and `latest` does not move, so "which build is that?" stays answerable. Pin the release family for a deployment; a `sha-` image is a way to run a specific diff --git a/docs/ENGINE.md b/docs/ENGINE.md index f4ac7b80d..5190191ac 100644 --- a/docs/ENGINE.md +++ b/docs/ENGINE.md @@ -458,7 +458,8 @@ higher cap is not lowered, and a server nobody registered is not created. The rewrite is the same flock and atomic rename, and a config that does not parse is left alone. -A driven Codex runs `-a on-request`, so every MCP tool call would otherwise +A driven Codex on a stack that sets `ENGINE_CODEX_APPROVAL_POLICY=on-request` +(the default is `never` since WI-2028) would have every MCP tool call stop at a permission prompt only a person may answer (WI-1634). The bundle therefore gives the estate's trusted servers `default_tools_approval_mode = "approve"`, the fork's per-server key that skips the prompt for that server @@ -783,14 +784,26 @@ than these routes. a label matches several options. Requires `sessions` (WI-917). A permission prompt (`kind` `permission` or `read-outside-cwd`) is answered only by a person: anyone else gets `403 person required` and nothing is - typed (WI-983, below). + typed (WI-983, below). One exception (WI-1831, widened by WI-2028, + `oversight_answer`): vogt-core may name an `overseer` (the engine id of the + oversight session it relays) on `/answer` and on `/input`. The engine + checks that session's role itself (role `oversight`, live, not the target) + and then lets it answer a `permission` prompt with any option or keys; a + `read-outside-cwd` prompt stays a person's. `overseer` is read only from + vogt-core's own credential. Answers and refusals are both audited + (`vogt::audit`, `event=session.permission_answer`, `via=answer|input`, + with the overseer, question and command excerpt). - `GET /api/sessions/changes[?since=N&ids=&limit=]` and `GET /api/sessions/wait-any[?ids=&until=&cursor=&timeout_s=]` — the overseer delta (WI-1246, WI-1672). `cursor` is a sequence number, or `now` to skip every event already in the ring and wait only for one published after the call starts. A timeout then returns that head as `next_cursor`. A numeric cursor past the head is treated as the start of the ring (the sequence - restarts when the engine does). Requires `sessions`. + restarts when the engine does). The change kinds are `ready`, `approval`, + `exited`, `errored`, `blocked`, `unblocked`, `hibernated`, `woken`, `budget` + and `done` (an autopilot agent declared itself done, WI-1518); the default + `until` is `ready`, `blocked`, `approval`, `exited`, `errored` and `done`. + Requires `sessions`. - `GET /api/sessions/sweep[?screen_lines=N&include_exited=true]` -> `SessionSweepEntry[]` — every live and hibernated session (exited ones only when asked) with the last N (default 8, at most 40) non-blank lines @@ -1087,6 +1100,16 @@ its own. `DELETE /api/sessions/:id` removes the guide with the brief, and the prompt sweep keeps a guide while its session is known, live or hibernated, exactly as it keeps the brief. +**Charters.** When the start request carries a non-empty `charter` (the +`## Charters` section of the brief, WI-2038) and the command is Codex, the +engine appends it to the `developer_instructions` override after the workspace +guidance, or adds one when there is none (`agent_cli::with_codex_charter`). +That body rides on every Codex request, so a charter over 4 KiB is replaced +there by a pointer to the brief's `## Charters`. An override that does not +parse, or a result over 120 KiB, is left alone with a warning in the engine +log; the brief still carries the charter. Other agent +CLIs ignore the field: the brief is how they get it. + **Project policy.** A project's `agent_policy` reaches a session as text in its brief, which vogt-core composes, and the engine writes no per-session hooks. One part of it is enforced before a session exists at all: @@ -1603,17 +1626,35 @@ checks decide what it can finish on its own (WI-926; the design is not the pod's brokered token, which is bound to a person. A plain shell keeps the pod's token. An agent a person starts by hand inside such a shell inherits the shell's token: the person started it. +- **A wake replays only the start's posture** (WI-2252). A posture granted + after the start (Shift+Tab in Claude Code, or `claude + --dangerously-skip-permissions` typed into a plain shell) is not in the + record, and is never replayed: the engine cannot tell a person's grant + from an agent's. At hibernation the engine reads Claude Code's footer + into `observed_permission_mode` on the record. When that was `bypass` and + the woken session has no bypass grant, its summary carries + `posture_lost_on_resume: true` (and the core's `session_sweep` row says so) + until a later hibernation sees the bypass back on the screen; the operator + re-grants it in the terminal. The record keeps only the `--settings` file + the first launch actually carried: a worker with none falls back to the + deployment's, an oversight session with none (or one that began as a + plain shell) is woken without the worker driven-session policy. - **Codex** (WI-1143) does not take Claude's `--settings` or `--permission-mode`. The engine maps the same three postures onto Codex flags, and unknown values are `400`: - - **default** (and a missing posture): `-a on-request -s danger-full-access`. - A `codex exec` launch omits `-a` (that subcommand rejects it) and keeps - the sandbox. - - **accept-edits** (`accept-edits` or `accept_edits`): `-a on-request -c - approvals_reviewer="auto_review" -s danger-full-access`. The `-a` pin is - what stops a user's `~/.codex/config.toml` `approval_policy = "never"` - from turning approvals off while the sandbox is `danger-full-access`; - the reviewer then answers the asks that pin produces. A `codex exec` + - **default** (and a missing posture): `-a -s danger-full-access`, + where `` is `ENGINE_CODEX_APPROVAL_POLICY` (config key + `codex_approval_policy`; one of `untrusted`, `on-failure`, `on-request`, + `never`; default **`never`**, WI-2028). With `never` a driven Codex + worker runs its commands without prompting, as the bypassed Claude Code + workers do; a stack sets `on-request` to restore per-command prompts + (which an oversight session may then answer). Any other value fails + engine startup. A `codex exec` launch omits `-a` (that subcommand + rejects it) and keeps the sandbox. + - **accept-edits** (`accept-edits` or `accept_edits`): `-a -c + approvals_reviewer="auto_review" -s danger-full-access`. The `-a` pin + stops a user's `~/.codex/config.toml` from choosing the approval policy; + under `on-request` the reviewer answers the asks that pin produces. A `codex exec` launch omits `-a` (that subcommand rejects it) and keeps the reviewer and the sandbox. - **bypass**: `--dangerously-bypass-approvals-and-sandbox` and nothing @@ -1834,6 +1875,21 @@ engine id (WI-912; the design is `docs/design/session-hibernation.md`). credentials: vogt-core's `session.wake` mints a new `VOGT_HTTP_TOKEN`. The engine mints a new broker grant. The history row and log continue. Waking a live session returns it unchanged. +- **Cold-wake compaction (WI-2037).** A wake with `fresh_prompt` (at most + 64 KiB) does not resume. The agent starts a new conversation, pinned to a + new conversation id so it does not collide with the old one. Its first + prompt is the summary, then the git facts of the session directory, then + the original brief (capped at 32 KiB), all written to the brief file. + After that, the record resumes the new conversation. vogt-core decides + when to send it: `cold_wake_compaction` is on (it is off by default), the + session has not set `cold_wake_opt_out`, its agent is Claude, Klaudia or + Codex, the session was hibernated longer than its provider's + `cold_wake_ttl_seconds`, and the last `economics.context_tokens` reading + is at least `cold_wake_threshold_tokens`. vogt-core builds the summary + from the last reply and the work item's newest hand-off, without a model + call. Every `session.wake` audit record carries the decision under + `cold_wake`. A hibernated session's summary carries its last + `economics` reading so that vogt-core can make this check. - **Restarts.** On `SIGTERM` the engine hibernates every session it can (`trigger: "shutdown"`) before the history drain, and an oversight session even when it is a plain shell. At boot, every record without a process @@ -1918,6 +1974,14 @@ agent may stop to announce its next step. So the engine re-drives it The summary's `autopilot` reads `false` once it is off. A nudge is input, so it restarts the quiet clock and nudges cannot stack up. +Declaring itself done is a change an overseer reads (WI-1518). The first time +the marker is seen in a turn, the engine publishes a `session-done` event +(`source`: `marker`), which `GET /api/sessions/changes` and `wait-any` report +as kind `done`, and the summary carries `done_at` (RFC 3339). A screen that +still shows the marker does not declare it again; `done_at` is cleared when +the next turn starts, and kept across a hibernation and a wake through the +hibernation record. + **When opencode is `ready`.** opencode draws no prompt glyph, so `ready` recognises its composer separately: the `┃` bar closed by a `╹▀` footer near the bottom of the screen, with no `esc interrupt` (which it shows only while a @@ -1942,8 +2006,10 @@ nothing) and acts at two levels: - **soft**, once, at `ENGINE_BUDGET_SOFT_PCT` (default 80) of a turn, context or whole-session clock limit: a `budget` change, and — only while the session is at its prompt — one typed line: `[vogt] budget: {what} ({n} of {limit}). Commit your work in - progress on your branch, write a hand-off comment (work_comment - kind=handoff), then stop.` Mid-turn, the line waits for the next tick that + progress on your branch and push it, write a hand-off comment (work_comment + kind=handoff: branch, head_sha, done, remaining, gates_run, blockers), then + stop.` A session that stops without one gets a minimal hand-off from the + core, built from its git state (WI-1833). Mid-turn, the line waits for the next tick that finds it ready. The prompt is read the way autopilot reads it: the session must not be blocked, its agent must be one the engine re-drives, and the prompt must have been quiet for the nudge settle time. A turn or context @@ -2019,9 +2085,14 @@ sets `resources.over_threshold` on any session at or over it. The core's #### Worktree build output A finished worker's build output is what fills the host disk: `target/`, -`node_modules` and `.venv` in a worktree nobody is using. With -`ENGINE_WORKTREE_CLEAN_ON_HIBERNATE` set, hibernating or stopping a session -whose directory is a linked git worktree removes those directories. A repo's +`node_modules` and `.venv` in a worktree nobody is using. Stopping or +removing a session whose directory is a linked git worktree removes those +directories, and so does removing a hibernated one. This is on by default +(WI-1750); set `ENGINE_WORKTREE_CLEAN_ON_HIBERNATE=0` to keep them. A +hibernation keeps them unless the variable is `true`: a hibernated session is +woken to carry on, and every engine shutdown hibernates every live session. A +session that exits on its own, without a stop, is not cleaned until it is +removed. A repo's main checkout is never touched — only a directory `git worktree add` created — and neither is a symlink or a file git tracks. A `target` directory counts only when a `Cargo.toml` sits beside it, so a directory of notes named `target` is @@ -2043,6 +2114,43 @@ what is free, the percent used, and `needs_attention` once that passes `ENGINE_DISK_WARN_PERCENT` (`85`). A deployment with no engine reports no row, which is a different fact from a volume that is fine. + +#### Session scratch, background processes and free space + +Claude Code keeps each session's scratchpad at +`/claude-///`, outside the worktree, and +nothing used to delete it: on 2026-10-18 scratch worktrees with their own Cargo +`target` (10–16 GB each) and load-test downloads held 292 GB and filled the +host (WI-1810). Unless `ENGINE_WORKTREE_CLEAN_ON_HIBERNATE` is false: + +- stopping or removing a session deletes its scratch directories; +- hibernating one keeps them but removes every Cargo `target` there (beside a + `Cargo.toml`, or holding `CACHEDIR.TAG`) and any top-level entry larger + than `ENGINE_SCRATCH_PRUNE_GB` (2). + +On hibernate, stop and remove the engine also `SIGKILL`s every process of its +user still carrying the session's `VOGT_ENGINE_SESSION_ID` — a `setsid` or +`nohup` job that left the process tree. + +A new session is refused with `507 Insufficient Storage` while the workspace +or temp volume has less than `ENGINE_MIN_FREE_PERCENT` (10) free; `0` turns +the guard off. A wake is let through. + +Cargo builds inside a session's scratch directory share one target per +repository: at start the engine writes +`/claude-//.cargo/config.toml` with `build.target-dir = +/` (default +`/claude-/cargo-target`; `off` disables). Cargo reads that file only +for builds under the scratch directory, so a session's own worktree keeps its +own target and workers never queue on one lock; concurrent scratch builds of +one repository do share its lock, which is the price of not building a fresh +10–16 GB target each time. A repository's own `.cargo/config.toml` or a +`CARGO_TARGET_DIR` wins. A repository's shared target over +`ENGINE_SHARED_CARGO_TARGET_MAX_GB` (40) is removed when a session stops. +Deleting a scratch directory also runs `git worktree prune` in the main +repository of each scratch worktree it held. Hibernation pruning weighs the +entries inside `scratchpad/` one by one, never `scratchpad/` as a whole. + #### Launch timing, logs and metrics A session's launch explains itself in the engine log and on `/metrics` @@ -2056,7 +2164,7 @@ fields are logged quoted: `event="launch.report"`). | `launch.first_output` | info, or warn at 10 s or more | the session's first byte of output | `session_id`, `name`, `launcher`, `first_output_ms` | | `launch.stage` | info | each stage the launch wrapper reports | `session_id`, `stage` (`login`, `secrets`, `bootstrap`), `ms`, `outcome`, and for `secrets` the `project` and `mode` | | `launch.secrets` | info (audit) | each secret project the launch read | `session_id`, `project`, `mode`, `ms`, `count`, `missing`, `secrets` (`VAR=SECRET_NAME` pairs, never a value) | -| `launch.report` | info, or warn when failed or 10 s or more | the wrapper hands over, or fails | `session_id`, `name`, `command`, `outcome`, `total_ms`, `error` | +| `launch.report` | info, or warn when degraded, failed or 10 s or more | the wrapper hands over, or fails | `session_id`, `name`, `command`, `outcome` (`ok`, `degraded`, `failed`), `total_ms`, `error` | `launcher` is `agent-auth` when the launch wrapper runs first, and `direct` otherwise. The wrapper (`vogt-agent-auth run`/`shell`) times its own stages @@ -3834,6 +3942,7 @@ can affect the forge; the same approval gate and core writeback policy apply. | `session.wait_any` | Operator-only | Unavailable: Blocks for up to ten minutes; a voice turn cannot wait that long. For agents over MCP/CLI/REST. | | `session.report_blocked` | Operator-only | Unavailable: An agent's report about its own session; for agents over MCP/CLI/REST. | | `session.report_unblocked` | Operator-only | Unavailable: An agent's report about its own session; for agents over MCP/CLI/REST. | +| `session.report` | Operator-only | Unavailable: An agent's typed report about its own session (progress, result, question, blocker); for agents over MCP/CLI/REST. | | `session.answer` | Operator-only | Unavailable: The assistant answers permission dialogs through its own approval flow; for drivers over MCP/CLI/REST. | | `session.token` | Operator-only | Unavailable: the session engine's own HTTP call to mint and revoke engine-started agent sessions' credentials; not a voice action. | | `session.sweep` | Voice-readable | Available: The voice roster — one digest line per session, the attention filter, a short screen. Accepts either id. This, not `session.list`, answers "what's going on". | @@ -3851,8 +3960,13 @@ can affect the forge; the same approval gate and core writeback policy apply. | `session.grant_list` | Operator-only | Unavailable: Lists grants to sessions; for the GUI and for agents over MCP/CLI/REST. | | `decision.ask` | Operator-only | Unavailable: An agent's question for a person; for agents over MCP/CLI/REST. | | `decision.answer` | Operator-only | Unavailable: A person's answer, made deliberately in the Inbox, never by voice. | +| `charter.get` | Operator-only | Unavailable: Standing instructions for agent sessions; for the GUI and agents over MCP/CLI/REST. | +| `charter.set` | Operator-only | Unavailable: Changes what every session brief on a scope carries; set it from the GUI or MCP, not by voice. | | `decision.list` | Operator-only | Unavailable: Lists decisions; for the GUI and for agents over MCP/CLI/REST. | | `decision.withdraw` | Operator-only | Unavailable: Withdrawing a decision; for the GUI and for agents over MCP/CLI/REST. | +| `watch.create` | Operator-only | Unavailable: A session arming its own wake trigger; for agents over MCP/CLI/REST. | +| `watch.list` | Operator-only | Unavailable: Lists armed and fired watches; for agents over MCP/CLI/REST. | +| `watch.cancel` | Operator-only | Unavailable: Disarming a watch; for agents over MCP/CLI/REST. | | `token.issue` | Operator-only | Unavailable: Issues credentials that must never enter model context. | | `token.list` | Operator-only | Unavailable: Admin credential inventory. | | `token.revoke` | Operator-only | Unavailable: Admin credential revocation. | @@ -3881,6 +3995,7 @@ can affect the forge; the same approval gate and core writeback policy apply. | `events.list` | Voice-readable | Available: Read the cursor-based event feed. | | `notifications` | Operator-only | Unavailable: GitHub-only view; voice uses inbox.list for complete attention coverage. | | `deployed.versions` | Operator-only | Unavailable: Not yet in the curated voice set; deploy-lane failures already reach voice through inbox.list. | +| `usage.board` | Operator-only | Unavailable: Not yet in the curated voice set; account limits and key spend are an operator view. | | `inbox.list` | Voice-readable | Available: List the normalized attention Inbox with coverage. | | `inbox.archive` | Confirmation-gated | Available after approval: Archive one normalized Inbox occurrence. | | `inbox.snooze` | Confirmation-gated | Available after approval: Snooze one normalized Inbox occurrence until a deadline. | @@ -4249,6 +4364,46 @@ such a deployment the manifest is cost without a boundary. **Default is off**: the brokered model stays the norm, and a deployment opts in per stack. Enabling it is an operator decision, not the engine's default. +**The engine does not depend on the secrets manager (WI-1812).** Agent auth is +never on the engine's start path. The engine binds its port and passes +`/readyz` whether or not the secrets manager (or anything behind it) is +reachable; what the engine itself needs — `ENGINE_TOKEN`, the stack secret, +the database location, the assistant keys — comes from the stack environment +or a mounted file, never from a live secrets-manager call. Credentials are +resolved per session, when the launch wrapper runs: + +- Each secrets-manager call in a launch (the login, each project read) is + bounded by `ENGINE_AGENT_AUTH_TIMEOUT` (seconds, default 10) and tried + once, and all of a launch's calls share `ENGINE_AGENT_AUTH_LAUNCH_BUDGET` + (seconds, default 20). A project read that times out or cannot connect is + not retried one secret at a time; the per-secret CLI fallback is only for a + read the API answered with an error. There is no retry loop. +- If the secrets manager cannot be reached — no identity, the login failed or + timed out, a required secret could not be read — the session **still + starts**, without the credentials it could not get. It prints an + "agent auth unavailable" warning in the terminal, exports the reason as + `AGENT_AUTH_UNAVAILABLE` (unset on a full launch), and `AGENT_AUTH_GRANTED` + names only what it actually holds. The identity is still dropped. +- The launch reports `outcome=degraded` (a `launch.report` warn line and the + launch metric). The engine keeps that on the session: `GET /api/status` + counts the live degraded sessions in `auth_broker.degraded_session_count` + and names the latest under `auth_broker.degraded` (`session_id`, `at`, + `reason`). Settings shows it on the auth-broker card. A degraded session + stops counting when it exits. +- Nothing is cached between launches, so once the secrets manager recovers + the next session gets its credentials with no engine restart. A degraded + session gets them by being restarted. +- Only a malformed manifest still fails a launch. `check`, `get` and `set` + keep failing loudly: they are explicit requests. +- `ENGINE_AGENT_AUTH_REQUIRED=1` no longer stops the container. The + entrypoint runs a bounded check in the background + (`ENGINE_AGENT_AUTH_BOOT_CHECK_TIMEOUT`, default 60 s) and logs + `agent service auth check passed` or `... FAILED`; the engine starts either + way. +- Deployment hooks: `vogt-lifecycle` starts the engine before it runs + `post-start.d`, so a hook that calls `vogt-agent-auth run` cannot hold the + engine back, and that call is itself bounded and degrades as above. + **Getting a secret after launch: the on-demand broker.** The one sanctioned late path is a *manifest-constrained fetch*, and it keeps the contract above intact: the identity never leaves the engine. A manifest line @@ -4299,6 +4454,11 @@ byte length — never value. `writable` widens blast radius only to the entries operator pre-authorised, and every overwrite is in the audit log. The same honest caveat applies. +**Approved actions (WI-2331, design only).** A proposed extension of the +grant pattern from credentials to engine-executed, person-approved +infrastructure actions (`vogt-agent-auth request`). Nothing is implemented; +see [`design/approved-actions.md`](design/approved-actions.md). + **Approved grants: one secret, one session, until a time (WI-973).** The manifest is fixed at deploy time, and changing it cycles the pod. A grant is the runtime counterpart: a person approves, in the Inbox, one named secret for diff --git a/docs/GITHUB_APP.md b/docs/GITHUB_APP.md new file mode 100644 index 000000000..1cb8f01df --- /dev/null +++ b/docs/GITHUB_APP.md @@ -0,0 +1,130 @@ +# Connecting Vogt to GitHub with a GitHub App + +Vogt's forge adapter reads GitHub (Actions runs, releases, issues, pull +requests, repository posture) and, with write-back enabled, writes to it. It +can authenticate in two ways: + +| | Personal access token | GitHub App | +|---|---|---| +| Config | `github_token_file` | `github_app_id`, `github_app_private_key_file`, optionally `github_app_installation_id` | +| Rate limit | 5,000/h, **shared** with everything else on that user | 5,000/h or more **per installation**, Vogt's alone | +| Acts as | the user | the App (`your-app[bot]`) | +| Notifications collector | yes | no — uses `github_token_file` if that is also set | + +Both are fully supported. A personal token is the quickest start. Use an App +when the sweep competes for rate limit with other tools on the same account, +when you want permissions narrower than a token's scopes, or when you want +writes attributed to a bot. + +When the App settings are set, they take precedence over `github_token_file` +for every GitHub read and write except notifications. If the key file is +missing or unreadable, Vogt logs a warning once and falls back to the token +file. + +## 1. Create the App + +On GitHub, go to **Settings → Developer settings → GitHub Apps → New GitHub +App**: under your user, or under an organisation's settings to make the +organisation own it. + +- **Homepage URL:** anything. Your Vogt URL is fine. +- **Webhook:** untick **Active**. Vogt polls, so it needs no webhook. +- **Where can this GitHub App be installed?** Choose **Only on this account**, + unless you will install it on accounts you don't own. + +Grant only the permissions for the features you use: + +| Permission | Access | Used for | +|---|---|---| +| Repository → Metadata | Read | Mandatory; repository identity | +| Repository → Actions | Read | `gh-actions` runs and jobs | +| Repository → Contents | Read | Releases, compare, the posture tree probe, clone on import | +| Repository → Administration | Read | Posture: vulnerability alerts and automated security fixes | +| Repository → Issues | Read | Issue sync. **Read and write** for write-back | +| Repository → Pull requests | Read | Pull-request sync. **Read and write** for write-back | +| Repository → Contents | Read and write | Only for `forge.publish`, which pushes the first branch | +| Repository → Administration | Read and write | Only for `forge.publish`, which creates the repository | +| Organization → Members | Read | Classifying authors as organisation members | + +Create the App. Note its **App ID** or **Client ID** from the App's page; Vogt +accepts either. Then, under **Private keys**, choose **Generate a private +key** and keep the downloaded `.pem` file. + +## 2. Install it + +From the App's page, choose **Install App** and pick each account or +organisation whose repositories Vogt should read. Select all repositories, or +only the ones your Vogt projects link to. + +One App can be installed on several accounts and organisations. Vogt discovers +the installation for each repository owner on first use, with +`GET /repos/{owner}/{repo}/installation`, then `/orgs/{owner}/installation` or +`/users/{owner}/installation`, and remembers it. A forge config that spans +several organisations therefore needs no per-owner settings. + +If the App is not installed on an owner, Vogt remembers that for an hour. In +that hour it reads the owner's repositories with `github_token_file`, or +unauthenticated if no token file is set, and makes no App calls for that +owner. An installation added later is picked up when the hour is up. + +To pin every request to one installation, set `github_app_installation_id`. +The installation ID is the number at the end of the installation's settings +URL (`.../settings/installations/`). + +**Publishing** (`forge.publish`) creates a repository before any repository +names an owner, so Vogt has to pick the installation: + +- If `github_app_installation_id` is set, Vogt uses that installation. +- If it isn't set and the App has exactly one installation, Vogt finds it + through `GET /app/installations` and uses it. +- If the App has several installations, set `github_app_installation_id`, + or publishing refuses rather than guess. + +On an organisation installation, Vogt creates the repository with +`POST /orgs/{owner}/repos`. + +## 3. Configure Vogt + +Deliver the private key as a file, the same way as the token file. Never +deliver it as an environment value. The two IDs are not secrets. + +```toml +github_app_id = "123456" # or the Client ID, e.g. "Iv1.abc..." +github_app_private_key_file = "/run/secrets/github_app_key" +# github_app_installation_id = 7654321 # optional: pin one installation +# github_token_file = "/run/secrets/github_token" # optional: notifications +``` + +Or, in the environment: `VOGT_GITHUB_APP_ID`, +`VOGT_GITHUB_APP_PRIVATE_KEY_FILE`, `VOGT_GITHUB_APP_INSTALLATION_ID`. + +With Compose, mount the key as a secret: + +```yaml +services: + vogt: + secrets: [github_app_key] + environment: + VOGT_GITHUB_APP_ID: "123456" + VOGT_GITHUB_APP_PRIVATE_KEY_FILE: /run/secrets/github_app_key +secrets: + github_app_key: + file: ./secrets/github_app_key.pem +``` + +## How tokens are handled + +Vogt signs a nine-minute RS256 JWT with the key (`iss` = the App ID or Client +ID) and exchanges it with `POST /app/installations/{id}/access_tokens` for an +installation token. It keeps each installation's token until five minutes +before it expires, which is about an hour. On a `401`, it mints a new token +and retries the request once. The rate budget and the ETag cache are kept per +installation, so one owner running low on budget doesn't pause the others. +Requests signed with the JWT, which look up installations and mint tokens, +have their own budget. Vogt checks it before sending one, so a low budget +stops those calls too. + +## Switching back + +Remove `github_app_id` or `github_app_private_key_file`, and set +`github_token_file`. Nothing else changes. diff --git a/docs/PYTHON_TEST_DISPOSITION.md b/docs/PYTHON_TEST_DISPOSITION.md index a6accf37b..caa6e3136 100644 --- a/docs/PYTHON_TEST_DISPOSITION.md +++ b/docs/PYTHON_TEST_DISPOSITION.md @@ -32,6 +32,7 @@ pinned by the `application/resolve.rs` tests in the Rust core (WI-1058). | File | Disposition | Where / why | |---|---|---| | test_agent_activity.py | ported | `collectors/agent_activity.rs` | +| test_agent_auth_degraded.py | unported | Degraded secrets-manager launch (`AGENT_AUTH_UNAVAILABLE`) is Python/shell session start, not yet in the Rust core. Deferred to WI-2588. | test_agent_auth_launch.py | retired | `engine/deploy/agent-auth.sh` launch timing. Shell, not the core crate. | | test_agent_cli_boot_contract.py | retired | `engine/deploy/agent-cli-install.sh` and `verify-agent-clis.sh`. Engine image boot, not the core. | | test_agent_clis.py | ported | Agent-CLI report service. The engine owns the mechanism; the core passes the report through. | @@ -45,17 +46,21 @@ pinned by the `application/resolve.rs` tests in the Rust core (WI-1058). | test_bootstrap_core_token.py | ported | Core-token adoption in `application/services/install.rs`. | | test_branch_binding.py | ported | `branches.rs` and the work-item branch binding service. | | test_bridge.py | ported | `adapters/mcp/bridge.rs`. | +| test_budget_handoff.py | unported | Budget-stop hand-off and resume is not yet in the Rust core. Deferred to WI-2588. | test_bundle_budget.py | retired | PWA first-screen bundle budget. `web/`, not the core. | | test_cargo_cache.py | retired | `engine/deploy/cargo-cache.sh` (WI-1256, done): Cargo cache survival on redeploy. Shell in the engine image, not the core crate; the Python test keeps driving the script. | +| test_charters.py | unported | Charter standing instructions are not yet in the Rust core. Deferred to WI-2588. | test_chats.py | ported | Chat operations in the application services, with the engine transport stood in. | | test_check_engine_spec.py | retired | `scripts/check_engine_spec.py` against the Rust engine routes. Engine contract, not the core. | | test_check_rust_version.py | retired | `scripts/check_rust_version.py`. A Python lint over the engine toolchain pin. | | test_check_workflow_policy.py | retired | `scripts/check_workflow_policy.py` over GitHub workflow YAML. CI policy, not product behaviour. | | test_checks.py | ported | CI-revision reads in the collectors and the project brief. | | test_ci_watch.py | ported | Watched-ref and deployed-version reads are in `application/deployed.rs`. The bound-branch announce is in `application/ci_watch.rs`: the feed event once per conclusion, staleness, the notice text, and the session nudge (notice, then Enter; none at a permission prompt). `test_a_sweep_announces_and_survives_an_announce_failure` is `a_failed_announce_does_not_fail_the_sweep` in `application/collect.rs`, driven by the `ci_watch::seam` failure switch. | +| test_claim_check.py | unported | Completion claims checked against git and the forge are not yet in the Rust core. Deferred to WI-2588. | test_claude_profile.py | retired | `engine/deploy/claude-profile.sh` and `mcp-bootstrap.sh` (WI-1306, done). Shell in the engine image, not the core crate; the Python test keeps driving the scripts. Named accounts on `session.start` are a sessions concern and are tested with that service. | | test_cli.py | ported | `adapters/cli.rs`. | | test_clone.py | ported | Clone service under `application/services`. | +| test_cold_wake.py | unported | Cold-wake compaction decision is not yet in the Rust core. Deferred to WI-2588. | test_collectors.py | ported | `collectors/` and `application/collect.rs`. | | test_config.py | ported | Precedence and default policy in `config.rs`. Generated-doc drift stays `scripts/gen_config_docs.py` until that generator is ported. | | test_connect.py | ported | Connect-document rendering in the application layer. | @@ -84,6 +89,8 @@ pinned by the `application/resolve.rs` tests in the Rust core (WI-1058). | test_front_door.py | retired | Engine front door (the `/api/vogt` proxy, PTYs, files). `engine/server`, not the core. | | test_git_story.py | ported | `git_story.rs` and `application/git_story.rs`. | | test_github.py | ported | `adapters/github.rs`. | +| test_github_app_auth.py | unported | GitHub App installation auth on the forge transport is not yet in the Rust core. Deferred to WI-2588. +| test_github_rate_budget.py | unported | Forge sweep rate budget and conditional requests are not yet in the Rust core. Deferred to WI-2588. | test_grants.py | ported | `application/grants.rs`. | | test_handoff.py | ported | Ported in `engine/core/src/application/services/handoff_test.rs` (16 tests, WI-1080): hand-off round-trip, comment order, the worker brief leading with the latest hand-off, every review-brief case, `review_of` start and refusal, and the three `from_handoff` cases (opens on the item's latest hand-off, refuses a different `work_item`, refuses an item with no hand-off). The upstream refusal is `a_handoff_comment_on_an_upstream_item_is_refused` in `upstream_truth_test.rs`. The validation case is split: fields on a note are refused by the service (`fields_on_a_note_are_refused`), and a hand-off kind without fields is refused by WI-1060's registry validator (`a_handoff_comment_without_fields_is_refused` in `engine/core/src/registry/validate.rs`). | | test_http.py | ported | `adapters/http/app.rs` (status, etag, validation, openapi). | @@ -117,6 +124,7 @@ pinned by the `application/resolve.rs` tests in the Rust core (WI-1058). | test_parity_normalise.py | retired | Normalisation helpers for the in-process parity script. `scripts/parity.py` owns normalisation now. | | test_place_metrics_cache.py | ported | `application/memo_test.rs` (WI-1054): the counts are shared across per-request contexts; a declared write, a rebuilt observed projection (`ObservedStore::latest_generation`) and an observed-side event each invalidate them; two data directories never share an answer; the key moves with each kind of change; a burst of callers costs one computation; a waiter takes only a computation that read the stores after it arrived; an answer `keep` refuses is asked for again (Python's provider-missing case, at the memo, since there is no monkeypatch to break `backlog`); the least recent stores are forgotten; the live Inbox half is read on every badge while the stored half is shared. Restore clears both memos. Not ported: the pooled read connections (`borrow_read`). Each Rust read opens its own rusqlite connection and is not serialised by a GIL. The engine's split read cap is engine-side and already on main. | | test_play_preflight.py | retired | `scripts/play_preflight.py` for the browser suite. `web/`, not the core. | +| test_policy_conflicts.py | unported | Landing-rule contradiction reading is not yet in the Rust core. Deferred to WI-2588. | test_policy_hook.py | retired | `engine/deploy/policy-hook.sh` and `install-policy`, the PreToolUse hook driven through the real script. Shell, not the core crate; the policy decision it asks the engine for is tested with the engine policy route. | | test_pr_work_item_edge.py | ported | `implemented_by` from pull requests, in forge sync and the work services. | | test_product_identity.py | retired | User-facing rename and legacy compatibility names across Python packaging. The Rust binary stays `vogt-core` for the overlap. | @@ -141,6 +149,7 @@ pinned by the `application/resolve.rs` tests in the Rust core (WI-1058). | test_session_last_reply.py | ported | Transcript reply reads in `adapters/transcripts.rs`. | | test_session_outcomes.py | ported | `collectors/session_outcomes.rs`. | | test_session_rename_remove.py | ported | Rename and remove in `application/services/sessions.rs`. | +| test_session_reports.py | unported | Typed session reports are not yet in the Rust core. Deferred to WI-2588. | test_session_sweep.py | ported | Sweep ordering in `application/session_reads.rs`. | | test_session_tool_descriptions.py | ported | MCP tool description strings generated from `registry/operations.rs`. | | test_sessions.py | ported | `application/services/sessions.rs`. | @@ -151,8 +160,10 @@ pinned by the `application/resolve.rs` tests in the Rust core (WI-1058). | test_unlinked_surfaces.py | ported | Unlinked-project honesty in `application/projects.rs` and the views. | | test_upgrade_data_volume.py | retired | Compose data-volume upgrade smoke. Deployment, not a core unit test. The migrator's upgrade-with-data cases cover the product half. | | test_upstream_truth.py | ported | Ported in `upstream_truth_test.rs`: a vogt-only change on an adopted item sends nothing, the unlinked refusal names both ways forward, a local-only item on an unlinked project comments and closes with nothing sent, a projectless item moves, each upstream issue counts once in the backlog and the board (adopted first), a relation between upstream issues is refused with the reason, `forge.link` refuses a project with no repository URL and is audited, a policy-`none` create on a linked project refuses and names every gate, and `local_only` creates work under `none` and on an unlinked project. The create and update write-throughs are ported (WI-1068): a create opens the issue and keys the overlay and ledger row by its subject, a failed create commits nothing, priority and effort on an upstream item are overlay-only, a label add writes through while a removal is refused as append-only, and title and body edits are refused as upstream truth. The comment, close and reopen write-throughs (WI-1780) are ported: `closing_and_reopening_write_through`, `an_open_to_open_move_on_an_upstream_item_sends_nothing`, `a_comment_posts_upstream_and_stores_no_local_row`, `a_handoff_comment_on_an_upstream_item_is_refused`, `a_failed_comment_and_close_raise_and_change_nothing`; a walk checks every hop's policy before moving anything (`a_walk_refused_mid_way_moves_nothing`). The board gaps (WI-1778) are fixed: an upstream-only issue is on the board once (`board_lists_an_upstream_only_issue_once`) and the cell cursor continues under its snapshot (`the_board_pages_with_stable_cursors`). WI-1779 was a misreading, now closed by a test: Python's `adopt` inserts a native row, so title, body and label-removal edits on an adopted item are local edits in both cores (`title_body_and_label_removal_on_an_adopted_item_are_local_edits`); the upstream-truth refusals apply to the un-adopted subject. Filed local-only because the vogt write-back policy is none. The import-with-consolidation link is `forge_import_clones_registers_and_consolidates_under_the_file_token`; the label refusal on an unlinked project is `a_label_edit_on_an_unlinked_project_is_refused`. Owner WI-1780. | +| test_usage_board.py | unported | Usage-board limits and key spend are not yet in the Rust core. Deferred to WI-2588. | test_view_paging.py | ported | Ranked backlog and bugs paging in `application/views.rs`. | | test_voice_capabilities.py | retired | Voice sidecar policy. `voice/` is a separate workspace. | +| test_watches.py | unported | Event watches and wake-on are not yet in the Rust core. Deferred to WI-2588. | test_work_get_many.py | ported | `work.get_many` batched chip read (WI-1224). Owner WI-1068. | | test_workflow.py | ported | Workflow machines in `decisions.rs`. | | test_workspace_disk.py | ported | `engine/core/src/application/workspace_disk_test.rs`: every case. status/sweep carry the disk row; the row is absent with no engine, engine unavailable, or a non-object answer; `engine.status` (ported in `application/services/engine_status.rs`, WI-1080) carries the row and says none without one. A row pydantic's `EngineDiskStatus` refuses (`{"used_percent":"x"}`, a wrong type, an unknown field) is no row on every surface, with pydantic's lax coercions reproduced (`"90"` and `90.0` are 90). Extra Rust cases pin the full `engine.status` result shape and the honest-unavailable `engine` message. Parity step `engine.status` (no engine) in golden 21c95f6. | diff --git a/docs/backlog-audit-2026-10-17.md b/docs/backlog-audit-2026-10-17.md new file mode 100644 index 000000000..5c6a1024a --- /dev/null +++ b/docs/backlog-audit-2026-10-17.md @@ -0,0 +1,458 @@ +# Vogt backlog audit — 2026-10-17 + +A read-only audit of the 283 open, in_progress and blocked items in project `vogt`. No item was changed: everything here is a proposal. + +Sources: +- Vogt MCP: `work_list`, `work_get`, `session_list`. +- `origin/main` and `origin/rust-core`, checked with commit greps by WI ref and `git cherry`. +- `gh` on TheDancingDeveloper-org/vogt. It shows **no open PRs**; landing is branch-push only. +- `docs/rust-port/gap-audit-2026-10-17.md`, on branch `docs/rust-port-gap-audit-2026-10-17`. + +Live sessions at audit time: WI-1067, 1071, 1072, 1080, 1749 and 1751. Every other linked session is dead; codex work was rescued to `salvage/2026-10-17/*`. + +## Summary + +| Class | Count (approx.) | +|---|---| +| done-but-open (close) | ~120 | +| stale / duplicate / superseded | ~30 | +| blocked | ~54 | +| in flight (live session or unmerged branch) | ~18 | +| ready | ~61 (about 15 of them ops-only or operator-gated) | + +The counts are approximate because some rows are "likely done, verify first". **Two systemic causes:** +1. The forge sync doesn't close a Vogt mirror when its GitHub issue closes. Seventeen or more gh# and WI rows mirror issues that closed on 2026-09-08 or earlier. +2. Landing a branch never transitions its item, so done work stays open or in_progress. That accounts for most of the ~120. + +**Both deserve a bug item. Closing items in batches alone will not keep the backlog clean.** + +### Proposed batch closes (done-but-open, with evidence in the tables below) + +- **Terminal attach series:** WI-121–131. +- **GitHub mirrors closed upstream:** WI-51, 73, 74, 84; gh#540, 467, 545, 613, 616, 620, 622, 626, 632, 639. +- **Call mode:** WI-960 and WI-965–970. +- **Earlier fixes:** WI-171, 177, 642, 643, 698, 774, 891, 952, 987, 1011, 1013, 1053, 1097, 1146, 1149. +- **Context management / H-series:** WI-1243, 1248, 1250, 1251, 1256, 1269, 1270, 1279, 1281, 1289, 1298, 1299, 1302, 1304, 1309, 1310, 1312, 1313, 1316, 1318, 1321, 1323, 1355, 1380, 1411, 1412, 1413, 1518. +- **Codex / CI / rail:** WI-1590, 1594, 1598, 1630, 1632, 1634, 1638–1641, 1645, 1647, 1664, 1667, 1670, 1680–1682, 1687, 1736, 1742, 1746, 1748. +- **Close after verifying:** + - WI-700, 1059, 1061, 1063, 1066, 1068, 1095, 1224, 1351, 1592, 1612. + - WI-1635 and WI-1644: close once deployed. + - WI-1671 and WI-1672: the Python side is done; move the Rust remainder to gap audit WP5–7. + +### Proposed stale / duplicate closes + +- **Stale:** WI-64, 66, gh#273, WI-181, 818, 852, 861, 995, 1010, 1014, 1093, 1260, 1280, 1529, 1661. +- **Superseded by WI-1151:** WI-1007, 1008, 1009, 1012, 1098. +- **Duplicates:** + - WI-1030 → WI-1007 + - WI-1018 → WI-1013 + - WI-1371 → WI-1359 + - WI-1621 → WI-1631 + - gh#583 → WI-695 + - WI-1574 → WI-1634 +- **Wont-do under the operator's "NO caps" ruling:** WI-894, and WI-895 unless the operator re-opens caps. +- **Umbrellas to re-scope or close:** gh#188, WI-826, WI-1265, WI-1517, WI-1603. +- **Partial, re-scope to the remainder:** + - gh#538 + - WI-1017 + - WI-1373 + - WI-1381 + - WI-1629: retarget to voice/Cargo.lock, which still has rustls 0.23.45 + - WI-1725: split the read-only onboard, which landed on rust-core, from consolidate + +### Ready: priority order + +1. **WI-1074** S8b sessions II: the Rust critical path, unstaffed. Mark it in_progress and staff it. +2. **WI-1064** P2.6 external adapters: critical path, unstaffed. +3. **WI-1725 remainder** (forge.onboard consolidate): critical path. +4. **WI-939** rotate the old prod agent token (p1 security, operator/Infisical). Run in parallel with **WI-1223** (rotate FORGEJO_TOKEN). +5. **WI-1000** grants security gap (p1). +6. **WI-1001** a blind Enter can approve a prompt (autopilot.rs). +7. **WI-1058** P1.9 application gaps: the branch exists, 5 ahead of rust-core, no session. +8. **WI-1324** workspace record: unblocks WI-1325, 1326, 1340 and 1330. +9. **WI-1341** GitOpReq::Land: the real head of the landing chain (1328 → 1330). +10. **WI-1520** session_report op: unblocks WI-1521, 1523 and 1524. +11. Then: + - **WI-1151**, p3 → p2: closes six items. + - **WI-1319**, then 1320. + - **WI-1744**, which unblocks I4. + - **WI-1747**, which feeds I5. + - WI-1331 and 1343, then WI-1327, 1329, 1322, 1334, 1253, 1254, 954, 799, 1658, 1660 and 1576. + +**Safe to run in parallel** (no shared files or goldens): +- **Engine:** WI-1324 (sessions.rs + new workspace.rs), WI-1341 (git.rs), WI-1001 (autopilot.rs) and WI-1331/1343 (agent-auth). Watch that WI-1324 and WI-1520 both touch engine sessions.rs. +- **Core:** WI-1319 (core/entities.py) and WI-1000 (grants). +- **Port:** WI-1074 (core services/sessions) and WI-1064 (core adapters), as long as neither re-records goldens (see the golden process below). +- **CI, tests and web:** WI-1658/1660 (workflows), WI-954 (integration.rs) and WI-1254 (web). + +**Serialize:** +- WI-1327 and WI-1326 both edit policy.rs. +- WI-1339 can re-cut demo baselines. +- Every parity slice (WI-1067/1069/1070/1071/1072/1073) waits for the golden lane. + +### Priority and effort mismatches + +- **Should drop:** + - p1 items blocked behind unbuilt work (WI-1325, 1326, 1328, 1340) while ready work waits. + - WI-1603: p1 umbrella whose fixes have merged → p2. + - WI-1661 and WI-852: obsolete p1 items. + - WI-1659: conditional → p4. + - Pipeline S4–S6 (WI-1651–1653): p1, but the design WI-1646 is unapproved. +- **Contradicted by rulings:** WI-894 and 895 are p1 against "NO caps". WI-893 is p1 with stale 10-05 data. +- **Under-ranked:** WI-1151 (p3, unblocks six items), WI-1000 and WI-1001 (security), WI-1324 (unblocks four), WI-1341. +- **No effort set:** WI-885–899 and WI-939–954. +- **Wrong state:** WI-1074 is open but has 26 rust-core commits. WI-1053 is DONE per the gap audit but still in_progress. About 15 P-items (WI-1043…1062) sit in `review` with no reviewer. + +### Bodies stale against current code + +- **WI-1629:** targets the engine, which is fixed; voice still has the vulnerable version. +- **WI-1725:** says it "unblocks consolidate", but a read-only port has landed. +- **WI-1017:** assumes an MSRV bump, against the stable 1.99 / floor 1.88 policy. +- **WI-996:** predates session_report_blocked, decision.ask and AUTOPILOT DONE. +- **WI-1260:** LiteLLM is already the router. +- **WI-1337:** superseded_by is handled in 21 files; it may already be done. +- **WI-1339:** the inbox row exists; only the PWA card is left. +- **WI-1315:** the docs rule landed; only the CI check is left. +- **WI-1317:** the design doc lives only on a branch while its children ship. +- **WI-891:** the ref was reused for terminal-resync commits. +- **WI-66 and WI-818:** name an old branch topology and an old SHA. +- **WI-1595:** PR #1007 was closed and never landed, yet WI-1527 depends on it. + +### Branch clutter (prune after verifying) + +`wi-973-engine-test-hang`, `fix/wi-995-codeql-ram`, `wi-1068-s3-work-ops-v2`, `wi-1066-s1-instance-auth`, `wi-1314-auto-unblock-b`, and the salvage branches for WI-1270, 1373, 1411, 1413 and 1224. + +### Operator decisions surfaced (not asked here) + +- **Make the rust-core Playwright step a gate** (I1). WI-1695 is marked done but conflicts with it. +- **WI-885:** decisions D1–D12. +- **WI-1646:** sign off the pipeline design. +- **WI-1637:** the requirements.toml pin. +- **WI-993:** Dependabot policy. +- **WI-1297:** six decisions. +- **WI-941:** prod hibernate policy. +- **WI-1342:** the brokered flip on dev. +- **WI-895:** caps. + +## Rust-port critical path to the swap (I1 → I7) + +rust-core tip `cf2d3ca6`. Live claude sessions: WI-1067, 1071, 1072, 1080, 1749, 1751. Codex sessions showing `attention=unknown` are dead; their work is on `salvage/2026-10-17/*`. + +**Feeding I1:** I1 (WI-1076) depends on S9 (WI-1075, in_progress). S9 is waiting on: +- **WI-1064** (P2.6 external adapters): in_progress, no live session. +- **WI-1074** (S8b sessions II): open, no live session, salvage branch exists. +- **WI-1725** (forge.onboard): open, not started. It unblocks `project.import consolidate=true`. + +**Nobody is working on these three, so they are the current bottleneck.** + +### Slices still showing GAPS + +| Slice | WI | Status | +|---|---|---| +| S1 | WI-1066 | Open. Branches `wi-1066-s1-instance-auth` and `-s1-report`, plus salvage. Needs review. | +| S2 | WI-1067 | Live. Branch `wi-1067-s2-gap-close` is 6 ahead, 8 behind. Re-records the golden. | +| S3 | WI-1068 | No session. Branches `wi-1068-s3-work-ops` and `-v2`. | +| S4 | WI-1069 | Core work landed (`9007667c`, `cf2d3ca6`). `wi-1069-golden-fix` is 1 ahead, 1 behind. Near review. | +| S5 | WI-1070 | No session. `wi-1070-s5-parity` is 2 ahead, 8 behind. Re-records the golden. | +| S6 | WI-1071 | Live. `wi-1071-s6-gaps` is 7 ahead, 0 behind. Re-records the golden. | +| S7 | WI-1072 | Live. The reviewer found one blocker. `wi-1072-s7-forge-gaps` is 5 ahead, 0 behind. | +| S8a | WI-1073 | No session. 4 ops have no parity step. Drift items WI-1316/1289/1672/1671/1255/1251 are still open. | +| P1.9 | WI-1058 | Open. Branch `wi-1058-application-gaps`. | +| P2.1/2.3/2.5 | WI-1059/1061/1063 | WI-1061's OpenAPI work landed (`54d16659`, `48753304`). WI-1742 is partly landed (`3314e9af`, `8f7c4c2e`, `901c0503`). Audit these: they may be done-but-open. | +| Schemas | WI-1748/1749 | WI-1748 step 1 landed (`360cd679`). WI-1749 is live. | + +### Order of the I-items + +1. **I1 WI-1076.** Still left: + - S9 has to close. + - PR #1045 (base `wi-1443-main-sync`) has to merge. + - Playwright in `rust-core.yml` is advisory (`|| echo`). Three specs fail against the Rust core: gui login-errors, terminalReplay reattach and demo stale-provenance. **Operator decision, never asked: make it a gate.** WI-1695 is marked done but conflicts with this. +2. **I2 WI-1077, PWA contract:** after I1. Its codex session died; work is on salvage. +3. **I3 WI-1078, stack image:** after I1 and WI-1751 (live). `engine/Dockerfile` still has the uv and CORE_IMAGE stages. Branch `wi-1078-i3-stack-image` exists. +4. **I4 WI-1079, scripts:** salvage only. Can run in parallel after I1. Also needs WI-1744. +5. **I5 WI-1080, test residue:** live, CI running. WI-1745 and WI-1747 roll into it. Can run in parallel with I2–I4. +6. **I6 WI-1081, drift closure and final golden refresh:** needs every S slice, plus I4 and I5. +7. **I7 WI-1082, the swap PR:** after I2, I3 and I6. + +Path: (WI-1074 ∥ WI-1064 ∥ WI-1725) → S9 WI-1075 → I1 (+ gate decision) → (I2 ∥ I3 + WI-1751) → I6 → I7. I4 and I5 run alongside until I6. + +Bookkeeping: about 15 P-items (WI-1043…1062) sit in `review` with no reviewer, so they need a close-or-review sweep. The gap audit rates WI-1053 DONE, but it is still in_progress. + +## Parity-golden contention (`tests/parity/golden/21c95f6/`) + +These five branches all rewrite `21c95f6/{cli,http,mcp,meta}.json`, and most also edit `tests/parity/script.json`: + +| Branch | Ahead/behind rust-core | Golden churn | Also touches | +|---|---|---|---| +| wi-1067-s2-gap-close | 6/8 | +1663/−46 | script.json, parity.py | +| wi-1069-golden-fix | 1/1 | +1847/−1 | — | +| wi-1070-s5-parity | 2/8 | +2058/−130 | script.json, parity.py, normalise.toml | +| wi-1071-s6-gaps | 7/0 | +1698/−219 | script.json, parity.py, normalise.toml | +| wi-1072-s7-forge-gaps | 5/0 | +254/−124 | script.json | + +rust-core re-recorded the same directory twice in a row (`43b227a2`, `cf2d3ca6`). + +Every re-record regenerates the whole transcript, because inserting a step shifts step order and sequential ids. So any two branches conflict on everything after their first inserted step. Re-recording on rebase fixes the conflict but quietly drops the other branch's steps whenever it ran against a stale script. The golden set is also fragmenting: `21c95f6-s4`, `21c95f6-s9`, `fe9d4db` (cli only), and `39b89c8`/`b108fa4`/`bd7bbb8` (no mcp). + +**Proposed process** +1. **Slice branches don't commit goldens.** A slice commits only its `script.json` steps and its normaliser/driver changes. CI records the Python implementation on the fly and compares against that. A `rust-core.yml` guard fails any non-lane branch whose diff touches `tests/parity/golden/**`. +2. **One serialized re-record lane.** Oversight or a `golden-refresh` job owns it. After each slice lands on rust-core, run `scripts/parity.py record --impl python` on all three transports into `golden/`. Commit that alone as `test(parity): golden @`. Merges are one at a time, so re-records never race. +3. **Shard the script by surface** (`script.d/.json`), each shard starting from its own fresh `init`, with goldens per shard. Then two slices only conflict when they touch the same surface. CI checks that a slice's golden diff stays inside its own shard. +4. **Keep one golden set.** Retire `-s4`, `-s9`, `fe9d4db`, `39b89c8`, `b108fa4` and `bd7bbb8` at the next lane re-record. `meta.json` must list cli, http and mcp, or CI fails. CI should also fail on MCP steps without params, which `parity.py:168` silently skips today. +5. **Now:** freeze golden commits on the five branches. Land their `script.json` changes least-behind first: WI-1069 → WI-1072 → WI-1071 → WI-1067 → WI-1070. Re-record in the lane after each merge, or once after all five. + +## Per-item classification + +Key: rc = origin/rust-core. "golden" = re-records tests/parity/golden/21c95f6. + +### WI-51 … WI-968 and GitHub mirrors + +| ref | class | evidence / notes | files (ready) | +|---|---|---|---| +| WI-51 | done-but-open | GH #491 closed; PRs #492/#493/#495 | — | +| WI-64 | stale | triage tracker, children mostly closed | — | +| WI-66 | superseded | gh#545 closed 09-08; the decision moved to WI-70/#564 | — | +| WI-73 / WI-74 / WI-84 | done-but-open | PRs #567 / #569 / #593 #594 #597 | — | +| gh#273 | stale tracking | its children are closed | — | +| gh#538 | partial | 17afc0ca (editor drafts); re-scope the rest | web | +| gh#540, #467, #545, #613, #616, #620, #622, #626, #632, #639 | done-but-open | closed on GH 2026-09-08; fixes 00a891cc, b747f1e7, cf69a33b, 16b6774f; Vogt mirrors did not sync | — | +| gh#188 | stale umbrella | voice epic; call mode landed; re-scope | — | +| gh#191 | ready (operator) | FCM client entry; needs Firebase access | android google-services.json | +| gh#583 | duplicate | merge into WI-695 | — | +| WI-121–131 | done-but-open | aeb7f86b, 144e3dd6, b046a06b, 12041472, 82bb8d1c, 3a5a9170, a31ce16f, d9d009b1, 141ea250, 303bd527 | — | +| WI-171 | done-but-open | c57e408f | — | +| WI-177 | done-but-open | PR #683 | — | +| WI-181 | stale (verify) | identity mode is live | — | +| WI-642 / 643 | done-but-open | 7ed38e05 | — | +| WI-695 | ready (operator, ops) | mint FCM key; absorb gh#583 | ops / Infisical | +| WI-698 | done-but-open | 93cb0fc2 | — | +| WI-700 | done-but-open (re-measure) | e6c386ae | — | +| WI-701 | ready (low; re-check logs first) | WS attach auth | engine auth | +| WI-774 | done-but-open | PRs #772, #829 | — | +| WI-799 | ready | deskd write-back gaps | services/forge*.py, adapters/github | +| WI-818 | stale | deploy of an old SHA | — | +| WI-826 | umbrella | check its children, then close | — | +| WI-852 | stale | merge-queue proposal obsolete under the push-branch policy | — | +| WI-859 | ready p4 (ops) | Cloudflare DNS via OpenTofu | ops | +| WI-861 | stale in_progress | no session or commit since 10-04 | — | +| WI-885 | blocked (operator D1–D12) | design 198e3c7d is on main | — | +| WI-886→887→888→889→890 | blocked | wait on WI-885 | — | +| WI-891 | done-but-open | push_api.rs:388 handles Lagged | — | +| WI-893 | blocked (operator); likely stale | — | — | +| WI-894 | conflicts with the "NO caps" ruling → wont_do | — | — | +| WI-895 | needs an operator call (NO caps) | — | — | +| WI-896 / 897 / 899 | ready (ops, node-b) | — | ops / grafana | +| WI-898 | ready (image) | gradle hygiene | pod image | +| WI-939 | ready p1 (security) | rotate the old prod agent token | Infisical | +| WI-940 | ready (ops) | prod /metrics scrape | ops prod compose | +| WI-941 | blocked (operator policy) | — | — | +| WI-952 | done-but-open | themes in styles.css | — | +| WI-954 | ready | integration.rs ~8433 sleep → poll | engine/server/tests/integration.rs | +| WI-960, 965–968 | done-but-open | 55b233f, dfa706ed, 0fb2edf5, 489fdb4f, db3f99fe | — | + +### WI-969 … WI-1253 + +| ref | class | evidence / notes | files (ready) | +|---|---|---|---| +| WI-969 | done-but-open | 01746e16 | — | +| WI-970 | done-but-open (verify) | 55b233fe, 7e1f8ebc, cff4ea3a | — | +| WI-973 | done-but-open (M1) | 91bcf41a, ae28883c; M2/M3 are tracked in WI-979/980; branch wi-973-engine-test-hang is stale | — | +| WI-977 | ready | — | opencode config | +| WI-979 | blocked | waits on WI-982 | — | +| WI-980 | blocked | waits on WI-979 | — | +| WI-982 | ready (L, p1) | separate uid | engine pty.rs, deploy | +| WI-987 | done-but-open | 5475453b; the remainder is WI-990 | — | +| WI-991 | ready | — | web terminal replay | +| WI-992 | ready p3 | cosign 3 | build/release workflows | +| WI-993 | ready (operator decision) | — | .github/dependabot.yml | +| WI-994 | blocked | waits on WI-995 | — | +| WI-995 | stale | fix/wi-995-codeql-ram 10-07, no PR; likely solved by runner placement | — | +| WI-996 | ready (p1, L, design) | body stale vs session_report_blocked / decision.ask / WI-1518 | engine, alertmanager | +| WI-1000 | ready p1 (security) | — | grants in services/sessions | +| WI-1001 | ready | blind Enter can approve a prompt | engine autopilot.rs | +| WI-1003 | ready p3 | L3 linkage | web board, core | +| WI-1007/1008/1009/1012/1098 | superseded by WI-1151 | Klaudia driven-agent gaps | — | +| WI-1030 | duplicate of WI-1007 | grok 400 | — | +| WI-1010 | stale (re-test) | wait_any with cursor (fe64958f) | — | +| WI-1011 | done-but-open | #901 | — | +| WI-1013 | done-but-open | Klaudia pin 3e03a65b | — | +| WI-1014 | stale (re-check) | likely fixed by WI-1094 parity | — | +| WI-1016 | ready p3 | egressy lane | — | +| WI-1017 | partial | 6193cab2; body assumes an MSRV bump (policy is stable 1.99 / floor 1.88) | engine Cargo.toml | +| WI-1018 | duplicate of WI-1013 | — | — | +| WI-1019 | blocked | waits on WI-990 | — | +| WI-1053 | done-but-open | gap audit says DONE; cmt_01M544C4965HEAQ36BDWGVWX93 | — | +| WI-1058 | ready / stalled | wi-1058-application-gaps 5 ahead of rc, no session | core brief.rs, services/sessions.rs, resolve.rs | +| WI-1059 | likely done | 8be93a4b (rc) | — | +| WI-1061 | likely done | 54d16659; HTTP gaps → WI-1742 | — | +| WI-1063 | likely done | c9aab9a2 (rc) | — | +| WI-1064 | in progress, unstaffed | cb976e35 (rc); **critical path** | engine/core adapters | +| WI-1065 | near done | 620d6aae, 0b8f65f2; consolidate waits on WI-1725 | — | +| WI-1066 | likely done | 341b1be0; check the s1-report branch | — | +| WI-1067 | in flight | live session; wi-1067-s2-gap-close | **golden** | +| WI-1068 | likely done | 3d15ee8f; the -v2 branch is stale | — | +| WI-1069 | in flight, no session | wi-1069-golden-fix 1 ahead | **golden** | +| WI-1070 | in flight, no session | wi-1070-s5-parity; 15 ops have no parity step | **golden** | +| WI-1071 | in flight | live session running | **golden** | +| WI-1072 | in flight | live session; review blocker | **golden** | +| WI-1073 | in progress, no session | 4 ops have no parity step | **golden** | +| WI-1074 | ready, unstaffed | 26 rc commits, salvage only; **critical path** | services/sessions*, grants | +| WI-1075 | in progress | S9; waits on 1064/1074/1725 | — | +| WI-1076–1082 | blocked chain | see the Rust-port section | — | +| WI-1079 | in progress, no session | 6ad7ce89 | scripts/ | +| WI-1080 | in flight | live session, CI | tests | +| WI-1093 | stale (verify) | 10-08 estate tooling | workspace | +| WI-1095 | done-but-open or needs a decision | 9cf4799e | — | +| WI-1097 | done-but-open | v0.7.8 #905/#907; ops env owed elsewhere | — | +| WI-1146 | done-but-open | c382cd3a, 573d874a | — | +| WI-1149 | done-but-open | 69693697, d7f70864 | — | +| WI-1150 | ready / stalled | needs a person-run on dev | docs/parity | +| WI-1151 | ready (p3 → p2) | closes 6 items | engine agent_cli.rs | +| WI-1223 | ready p1 (ops) | rotate FORGEJO_TOKEN | — | +| WI-1224 | likely done | e6c386ae, 77361235 (v0.7.10); verify the Caddy half | — | +| WI-1243 | done-but-open | 6bf47108, a3d2df95; the Rust remainder is WI-1058 | — | +| WI-1248 | done-but-open | 70bc3e0a, a49bac80, ba27a5d3 | — | +| WI-1250 | done-but-open | e19fe6c3, 637d6b51, dd6ea745 | — | +| WI-1251 | done-but-open | ca2c69dc | — | +| WI-1253 | ready (p2, S) | — | core events / sweep | + +### WI-1254 … WI-1524 + +| ref | class | evidence / notes | files (ready) | +|---|---|---|---| +| WI-1254 | ready | GUI refresh debounce | web session store / SSE | +| WI-1256 | done-but-open | e1201c76, 3199b57a | — | +| WI-1260 | stale | LiteLLM is already the Codex default (WI-1147); re-scope or close | — | +| WI-1263 | ready p3 | envsubst in runner images | runner Dockerfiles | +| WI-1265 | blocked umbrella | close after WI-1266–1268 | — | +| WI-1269 | done-but-open | 9cf4e91b, 6720984d | — | +| WI-1270 | done-but-open | 4fb96621, c1248915, 89e4c05a | — | +| WI-1273 | ready | ENGINE.md:1599 accept-edits | agent_cli.rs, docs/ENGINE.md | +| WI-1279 | done-but-open | 712b790b | — | +| WI-1280 | stale | review whose output became 1281/1284/1297 | — | +| WI-1281 | done-but-open | 96560109, 0b96dfb2, 923d4a64 | — | +| WI-1284 | ready | — | engine transcripts index, sessions.py | +| WI-1289 | done-but-open | 1986e3a6, 645b2819 | — | +| WI-1291 | ready (ops) | GCP runner TMPDIR | runner config | +| WI-1297 | review | report uncommitted; 6 operator decisions pending | — | +| WI-1298 | done-but-open | 69d744a6, e087db20, 2c6e83d1 | — | +| WI-1299 | done-but-open | 68385649, ccf818f5 | — | +| WI-1302 | done-but-open | 24c5be31 | — | +| WI-1304 | done-but-open | 2c5d3602, 885ccaa0 | — | +| WI-1309/1310/1312/1313 | done-but-open | 1e5d06c3 / eda031ba / 62bf6abb / 58399ccd | — | +| WI-1314 | stalled in flight | wi-1314-auto-unblock-c has 1 commit off main; no session or PR | sessions.rs, autopilot.rs | +| WI-1315 | ready | docs rule landed; CI check not built | .github/workflows | +| WI-1316 | done-but-open | 9d2232cf, bd83b433 | — | +| WI-1317 | in flight | design doc only on wi-1317-harness-design; land it or close | docs/design | +| WI-1318 | done-but-open | 8b822484 | — | +| WI-1319 | ready | verdict comment kinds | core/entities.py, work comments | +| WI-1320 | blocked | waits on WI-1319 | — | +| WI-1321 | done-but-open | 1bbe3771, 6fdb80f6 | — | +| WI-1322 | ready | decision change kind | engine sessions changes | +| WI-1323 | done-but-open | 6f5526e2 | — | +| WI-1324 | ready (unblocks 4 items) | workspace record | engine sessions.rs, new workspace.rs | +| WI-1325/1326/1340 | blocked | wait on WI-1324 (1325 overlaps 1304, so re-scope) | — | +| WI-1327 | ready | push/PR-open rule | engine policy.rs | +| WI-1328 | blocked | waits on WI-1341, WI-1319 | — | +| WI-1329 | ready | dispatch enforcement | services/sessions.py | +| WI-1330 | blocked | waits on WI-1328 | — | +| WI-1331 | ready | askpass token | engine agent-auth broker | +| WI-1333 | ready (docs) | — | docs/ENGINE.md | +| WI-1334 | ready | ci.run_hung | CI collector, inbox.py | +| WI-1335 | blocked | waits on WI-1334 | — | +| WI-1336 | ready p3 (docs) | — | docs | +| WI-1337 | ready (verify; may be done) | superseded_by already in 21 files | sessions.py | +| WI-1338 | ready | local_only auto-chore | sessions.py | +| WI-1339 | ready | inbox row exists; PWA card only | web inbox (demo baselines) | +| WI-1341 | ready | GitOpReq::Land | engine git.rs | +| WI-1342 | ready (operator-gated) | brokered flip | ops | +| WI-1343 | ready | `vogt-agent-auth with` | engine/deploy/agent-auth.sh | +| WI-1344 | blocked | waits on 1342, 1331, 1343 + one week on dev | — | +| WI-1351 | done-but-open (verify) | 80df588b, 115a170f | — | +| WI-1355 | done-but-open | 994b8031, d734c08b, 761e0453 | — | +| WI-1359 | blocked (infra) | pod userns | — | +| WI-1371 | duplicate of WI-1359 | — | — | +| WI-1373 | partial | 0365057f, b108fa4b; re-scope the analysis remainder | — | +| WI-1380 | done-but-open | 319bb466, 56001df2, a15d48c8 | — | +| WI-1381 | partial | 73ad343f; STT/TTS latency metrics remain | voice | +| WI-1411 | done-but-open | 64466141, 04e316a7 | — | +| WI-1412 | done-but-open | b0bffb4d | — | +| WI-1413 | done-but-open | 9f5b5d34, a66205b8, b69f3bde; check wi-1413b (+3) | — | +| WI-1517 | blocked umbrella | children open | — | +| WI-1518 | done-but-open | 7f4a4d11 | — | +| WI-1520 | ready | session_report op | engine sessions.rs, core, MCP parity | +| WI-1521 | blocked | waits on WI-1520 | — | +| WI-1523 | blocked | waits on WI-1520 | — | +| WI-1524 | blocked | waits on WI-1520, WI-1521 | — | + +### WI-1525 … WI-1751 + +| ref | class | evidence / notes | files (ready) | +|---|---|---|---| +| WI-1525 H8 compact sweep | ready* | no commits | services/sessions.py sweep | +| WI-1526 H9 oversight model trial | ready (operator decision) | an experiment | brief/profile config | +| WI-1527 H10 ACP pilot | blocked | waits on WI-1595 codex-acp | engine driver | +| WI-1528 H11 brief cleanup | ready* | — | engine prompt templates | +| WI-1529 H12 tracker hygiene | stale | branch from 10-14, no PR, session dead; this audit overlaps it | — | +| WI-1530 H13 aidevenv docs | ready | lives in the aidevenv repo | aidevenv TOOLING.md | +| WI-1574 | superseded by WI-1634 | b3962fed; per-tool annotations may remain | — | +| WI-1575 | ready* | Codex economics | agent_cli.rs, session_economics.py | +| WI-1576 | ready* | Codex ready/done from rollout | autopilot.rs, pty.rs | +| WI-1577 | blocked | waits on WI-1576 | — | +| WI-1578 | ready* | Codex hibernate/wake | engine hibernate | +| WI-1579 | ready (tiny) | AGENTS.md 32 KiB budget | AGENTS.md | +| WI-1590 | done-but-open | f837017e, 943d4c7d | — | +| WI-1592 | done-but-open (verify on dev) | 8772a3a5 | — | +| WI-1593 | ready* (ops) | bwrap userns | pod securityContext | +| WI-1594 | done-but-open | c3225e4c | — | +| WI-1595 | stalled | PR #1007 closed, not on main; verify | agent-versions.env, Dockerfile | +| WI-1597 | blocked | waits on the WI-1603 fixes being deployed to dev | — | +| WI-1598 | done-but-open | 7f3a8cdf | — | +| WI-1599 | blocked | waits on the WI-1635 deploy to dev | — | +| WI-1602 | blocked | operator grant on dev | — | +| WI-1603 | blocked umbrella | fixes merged (#1012–#1015); re-run the bar on dev | — | +| WI-1612 | likely done | #1013 Klaudia pin; verify | — | +| WI-1621 | duplicate | merge into WI-1631 | — | +| WI-1629 | done for engine (8771d424) | voice/Cargo.lock still has rustls 0.23.45; retarget to voice | voice/Cargo.lock | +| WI-1630 | done-but-open | ed24df39, 5941d8ab | — | +| WI-1631 | ready (ops) | LiteLLM groups | litellm config | +| WI-1632 | done-but-open | e9202450 | — | +| WI-1634 | done-but-open | b3962fed | — | +| WI-1635 | done-but-open (deploy pending) | 194f1719 | — | +| WI-1637 | ready (operator decision) | requirements.toml pin | — | +| WI-1638–1641 | done-but-open | #1012–#1015 | — | +| WI-1644 | done-but-open | #1019 (bd7bbb8e); close once it is on prod | — | +| WI-1645 | done-but-open | #1020 + fe9d4db9 | — | +| WI-1646 | in review | branch design/build-deploy-pipeline-wi1646, no PR; needs operator sign-off | docs/design | +| WI-1648 | blocked / ready | waits on WI-1646 sign-off | vogt-estate | +| WI-1651 | stale in flight | session dead, no branch | vogt-estate | +| WI-1652/1653/1656/1657 | blocked chain | 1651 → 1652 → (1653, 1656) → 1657 | — | +| WI-1654 | blocked | operator: Infisical/OIDC | — | +| WI-1655 | blocked | waits on WI-1654 | — | +| WI-1647 | done-but-open | #1021 | — | +| WI-1658 | ready | CodeQL off bigmem | .github/workflows/codeql.yml | +| WI-1659 | blocked | conditional on WI-1658 → p4 | — | +| WI-1660 | ready | publish-runner trim | .github/workflows/build.yml | +| WI-1661 | stale | 10-15 plan; close | — | +| WI-1663 | ready* | GUI dispatch settings | web settings | +| WI-1664 | done-but-open | #1027 | — | +| WI-1667 | done-but-open | #1034 | — | +| WI-1670 | done-but-open | #1039, #1041 | — | +| WI-1671 | done-but-open (Py, #1036) | Rust remainder is gap audit WP7 | — | +| WI-1672 | done-but-open (Py, #1035) | Rust remainder is WP5/6 | — | +| WI-1680, 1681 | done-but-open | #1046 | — | +| WI-1682 | done-but-open | #1048 | — | +| WI-1687 | done-but-open | #1050 (rc) | — | +| WI-1690 | blocked | waits on I7 + soak | — | +| WI-1725 | partial | 54be0a78 read-only onboard on rc; consolidate is not done; split the item | engine/core forge | +| WI-1736 | done-but-open | #1068 | — | +| WI-1742 | done-but-open | #1070 (rc) | — | +| WI-1744 | ready | in-process load driver | engine/server/tests/load.rs, scripts/load.py | +| WI-1745 | blocked | waits on service ports (WI-1725 and others) | — | +| WI-1746 | done-but-open | #1071 | — | +| WI-1747 | ready | front-door audit rows | engine/server front-door tests | +| WI-1748 | done-but-open | #1072 (rc) | — | +| WI-1749 | in flight | live session, gates running; branch wi-1749-schemas-from-models; retires the recorded schema table | registry, models.rs | +| WI-1751 | in flight | live session running; gates I3 | engine/core CLI | + +\* no commit or PR matched by WI ref; I did not read the body. diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 000000000..eb7940fa5 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,93 @@ +# CI: pull requests, the merge queue, and `main` + +Vogt's CI does two different jobs. A pull request gets fast feedback on what +it changed. The merge queue proves that the exact tree `main` will become still +passes everything. Before WI-1886 every PR ran the whole suite (20–60 minutes), +and each merge forced sibling PRs to rebase and run it all again. + +## What runs where + +| Event | `ci.yml` | `codeql.yml` | `runner-policy` | `dependency review` | +|---|---|---|---|---| +| `pull_request` | only the halves the diff touches (below) | **does not run** | full | reviews the PR's dependency changes | +| `merge_group` (the queue) | **everything** | **does not run** | full | posts the context; the review already ran on the PR | +| `push` to `main` | everything | **does not run** | full | — | +| `push` of a `v*` tag (beta, prod) | — | every language | — | — | +| `workflow_dispatch` | — | every language, at `ref` if given | — | — | + +### Path scoping on a pull request + +`ci.yml`'s `classify changes` job reads `git diff` against the merge base: + +| Changed paths | Runs | +|---|---| +| `.github/workflows/`, `Dockerfile`, `.dockerignore`, `deploy/`, `.gitignore` | everything, as in the queue | +| `src/`, `tests/`, `scripts/`, `pyproject.toml`, `uv.lock`, `config.example.toml` | Python lint/mypy/pytest on 3.13, and the core-alone job | +| `engine/`, `docs/ENGINE.md` | the engine job's **rust** half: fmt, rust-version, cargo audit, clippy, nextest, doctests | +| `web/` | the engine job's **web** half: pnpm audit, typecheck, vitest, demo Playwright, PWA build, bundle budget | +| `voice/` | the voice job | +| `mobile/` | the android job | +| anything else (docs, markdown) | nothing: the `ci` gate passes on skipped halves | + +Only the queue and `main` run these: + +- Python **3.11**, the declared floor. A PR runs 3.13 only. +- the core-alone job when no Python changed. It deletes `engine/`, `web/` and + `mobile/` before it tests, so on a PR only a Python change can affect it. +- a web-only change rebuilt into the engine. The crate embeds `web/dist`, but + a web-only PR does not run clippy or nextest against the new bundle. + +### CodeQL + +CodeQL runs only when a release is cut, as an operator decision. It does not +gate pull requests, the merge queue, or `main`, so its `analyze ()` +jobs are not required checks. A `v*` tag push (beta and prod) runs it. So does +a `workflow_dispatch`: the dev deploy lane passes the commit it deploys as the +`ref` input, and checkout uses that SHA. Every such run analyses all three +languages. The job names are unchanged. + +### Concurrency + +Every workflow that runs on PRs cancels in progress per ref (`github.ref`). +A force-push to a PR therefore cancels the superseded run. Each queue entry has +its own `gh-readonly-queue/main/pr-N-` ref, so queue entries never cancel +each other. The queue removes an entry it no longer needs. + +### Required checks must run in the queue + +A required context that nothing posts on `merge_group` leaves every queued +entry waiting until the queue times it out. `scripts/check_workflow_policy.py` +(run by `runner-policy`) fails when a check listed under SECURITY.md's +"required checks" has no `merge_group` producer. + +### Runners + +Queue runs use the same labels as PR runs: `self-hosted`, plus `docker`, +`publish` or `bigmem` where a job asks for one. No vogt workflow targets the +GCP rust-heavy or spot labels. The queue adds full runs, but PRs now skip +everything they do not touch, and a merge no longer re-runs every open PR. +CodeQL no longer runs in the queue. Its single `bigmem` slot serialises the +three languages within a release run. + +## Landing + +`land-pr` (workspace `.agent-tools/bin/land-pr`) detects a merge queue from +the base branch's rulesets. It enqueues with `gh pr merge --auto` and follows +the entry's `merge_group` runs. On a failed run it prints the failing job's +log and exits 1. The PR is never rebased by hand while it is queued. + +## Turning the queue on (operator) + +The workflows support a queue but do not require one. Until a ruleset with a +`merge_queue` rule exists on `main`, PRs merge exactly as before, with the +narrower PR checks. The exact ruleset is in the WI-1886 hand-off. In +outline: one branch ruleset on `main` with a `merge_queue` rule +(method rebase, build concurrency 3, group size 1–3, wait 5 minutes, +`ALLGREEN`, check timeout 60 minutes) and the same required checks as today +(`ci`, `runner-policy`, `dependency review`). + +Once the queue is on, `push` to `main` re-runs the full suite on a commit the +queue has already tested. It is kept deliberately for now: `build.yml` and +`e2e.yml` key off it, and it is the full check for anything that reaches `main` +without the queue. Dropping `ci.yml`'s `push` trigger is a follow-up once the +queue has proved itself. diff --git a/docs/design/approved-actions.md b/docs/design/approved-actions.md new file mode 100644 index 000000000..015bb7387 --- /dev/null +++ b/docs/design/approved-actions.md @@ -0,0 +1,583 @@ +# Approved actions: engine-executed infrastructure steps (WI-2331) + +Status: **design only**. Nothing here is implemented. Where this note and +[`ENGINE.md`](../ENGINE.md) or [`API.md`](../API.md) disagree, those describe +what exists. This note applies the [approved-grants](oversight-grants.md) +pattern (WI-973) to *actions* instead of *credentials*: a person approves one +exact operation, and the engine carries it out. + +## Problem + +A driven-session worker often reaches a step the operator has already agreed +to: apply a Tailscale ACL change, deploy a Komodo stack, merge a Forgejo pull +request in `indexarr/ops`. The worker can't do it: + +- The driven-session classifier (`engine/deploy/driven-session-settings.json`) + treats Komodo writes, Tailscale policy and Forgejo merges outside the + "Own Green PR Merge" rule as protected-scope changes, and refuses them. That + refusal is correct. Nothing in the transcript proves a person approved the + step, so the classifier can't tell an approved apply from an unapproved one. +- The approval lives in chat. Oversight then runs the step itself with broad + credentials, such as the operator's personal Forgejo token. The worker that + holds the context stops, and the action is audited only as "oversight did + something". +- A credential grant (WI-973) doesn't fit. It hands the worker a secret that + can do far more than the one approved step, and the classifier still refuses + the protected use of that secret. + +Nothing in the system can express "approved: this exact action, with these +parameters, for this work item, once". + +## The shape + +```text +worker (driven) core approver engine + | vogt-agent-auth request ---- broker token --------------------------------->| + | |<-- POST /api/agent-auth/action-requests (engine relays, adds session) --| + | | action_requests row (pending), audited | + | |-- Inbox entry ----------->| | + | session_report_blocked | | | + | |<-- action.decide (approve) | | + | |-- POST /api/actions/{id}/execute (stack secret) -->| + | | | preconditions | + | | | execute (engine cred)| + | | | verify / rollback | + | |<------------------------ outcome ------------------| + | | row: succeeded | failed | rolled_back, audited | + |<-- wake + outcome ----------------------------------------------------------| +``` + +The split is the same as for grants. The core owns the **decision**: who +asked, the exact parameters, who approved, the item, and the audit trail. The +engine owns the **effect**: it holds the action credentials, runs the +executor, and is the only process that touches the target system. The worker +never holds a credential that can perform the action. + +## 1. Data model + +### 1.1 Action types (code, not data) + +An action type is a Rust module in the engine, registered under a stable dotted +name. It's code, not configuration, because its preconditions, verification +and rollback are code. Each type declares: + +| Field | Meaning | +| --- | --- | +| `name` | `forgejo.merge`, `komodo.deploy`, `proxmox.apply`, `ansible.apply`, `tailscale.acl.apply`, `tailscale.device.tag`. | +| `params_schema` | A JSON schema. Every field is required or has a stated default, `additionalProperties: false`. Free text is allowed only in `reason`. | +| `risk` | The type's base risk, `standard` or `person_only`. A request's *effective* risk can only be raised from it, never lowered (§1.5). Every `tailscale.*` type is `person_only`, and so is any request whose target matches the deployment's prod pattern. | +| `classify(params, plan)` | Computes the request's effective risk and `destructive` flag from its parameters and, where the type has one, the computed plan (§1.5). | +| `target(params)` | The canonical target key the anti-splitting rules use (§1.6), e.g. `komodo:vogt-dev`, `forgejo:indexarr/ops#98`, `proxmox:root=proxmox`. | +| `credential` | The engine-held credential the executor uses, by name (§3.5). | +| `preconditions(params)` | Read-only checks the engine runs at execute time (§3.1). | +| `execute(params)` | The one change. | +| `verify(params, before)` | A read that proves the change took effect (§3.2). | +| `rollback(params, before)` | Puts the target back to the captured `before` state, or `none` with a stated reason (§3.3). | +| `deadman_seconds` | How long the change may stand unconfirmed before the engine rolls it back automatically. `0` means no dead-man. | + +`GET /api/actions/types` lists the registered types with their schemas, risk +and whether this deployment has their credential configured. A type without a +credential is listed as `unavailable`. A request for it is refused when it is +made, not when it is approved. + +### 1.2 Action requests + +Core table `action_requests`, in the next free migration: + +| Column | Meaning | +| --- | --- | +| `id` | `act_…` ULID. | +| `action` | The type name. | +| `params` | Canonical JSON (keys sorted, no whitespace), validated against the type's schema at request time. | +| `params_digest` | `sha256(action ‖ 0x00 ‖ params)`. The approval binds to this digest (§6.2). | +| `work_item` | `WI-n`. Required. The item must exist and be open. | +| `target` | The type's `target(params)` key, stored so earlier outcomes on the same target can be found (§1.6). | +| `risk`, `destructive` | The effective risk and destructive flag computed by `classify` at request time and again at execute time (§1.5). | +| `supersedes` | Optional `act_…` id of an earlier denied, failed or rolled-back request on the same item and target that this one replaces. It must name such a request, or the request is refused (§1.6). | +| `requester_session` | The `ses_…` id, taken from the engine relay, never from the body. | +| `requester_principal` | The authenticated identity_ref of the relay. | +| `reason` | Shown to the approver verbatim as untrusted text. ≤ 2000 characters. | +| `state` | `pending` → `approved` → `executing` → `succeeded` \| `failed` \| `rolled_back`; or `denied`, `withdrawn`, `expired`. | +| `approver`, `approval_kind` | A person's identity_ref and `person`, or an oversight session and `delegation:`. | +| `decided_at`, `expires_at` | A pending request expires after `ttl_seconds` (default 86 400, at most 604 800). An approval must be executed within 15 minutes, or it expires as well. | +| `outcome` | The executor's structured result: precondition results, `before` snapshot digest, the change's id (merge SHA, Komodo update id, ACL ETag), verify result, rollback result. No secret, ever. | + +A request is **single-use**. `approved` moves to `executing` in one +conditional update (`WHERE state = 'approved'`), so a second execute call finds +nothing to run. + +### 1.3 Approvals + +An approval is the decision on one request, recorded on the row: who, when, +by which path (person, or a named delegation), and the `params_digest` the +approver saw. A person approves in the Inbox, which shows the exact +parameters. They are rendered from the stored canonical JSON, not from the +reason. There is no "approve with edits". To change a parameter, the person +denies and the worker requests again. + +### 1.4 Standing delegations + +Core table `action_delegations`, managed only by a person (`action.delegation_set` +refuses every agent principal): + +| Column | Meaning | +| --- | --- | +| `id` | `dlg_…`. | +| `project` | The Vogt project the delegation applies to. A request is in scope when its `work_item` belongs to this project. | +| `actions` | The action types oversight may approve. Only `forgejo.merge`, `komodo.deploy` (non-prod) and `proxmox.apply` are delegable. Naming a `person_only` type is refused, and so is `ansible.apply` until a person decides it is delegable. | +| `constraints` | Per type, a narrowing of the parameters, e.g. `forgejo.merge: {repo: ["indexarr/ops"], style: ["rebase"]}`, `komodo.deploy: {stack: ["vogt-dev"]}`, `proxmox.apply: {root: ["proxmox"]}`. A constraint can only narrow the schema. | +| `approvers` | Which oversight sessions may approve: `any_oversight` (the WI-957 role, which only a person can assign) or listed `ses_…` ids. Either way, an overseer approves only requests from workers it started itself (below). | +| `max_per_day` | A rate cap on delegated approvals. Default 20. | +| `expires_at` | At most 30 days. No delegation is permanent. | +| `created_by`, `reason` | The person who set it, and why. | + +An oversight session approves a request through a delegation with +`action.decide` like a person would. The core checks every one of these, and +any failure means the request needs a person: + +- **Own workers only** (decided 2026-10-24). The requester session's + `started_by_session` is the approving oversight session, read from the + engine's session record, never from the request. An overseer can't approve + a request from a worker another overseer started, from a session a person + started, or from itself. If the starting overseer is gone (stopped, + forgotten, or demoted from the oversight role), nobody inherits its + delegation: the worker's requests go to a person. +- The approver is an oversight session the delegation lists. +- The request is in the delegation's project, and its type and parameters are + within the constraints. +- The request's **effective** risk is `standard` and it is not `destructive` + (§1.5). That covers the type, the prod pattern (`*prod*`, the same rule the + Komodo MCP uses) and the computed plan. +- The request is not **routed to a person** by the anti-splitting rules (§1.6). + +### 1.5 Destructive requests + +"Destructive" is computed per request, not per type. Each executor's +`classify` runs at request time, and again at execute time against the live +plan. It sets `destructive: true`, and with it `risk: person_only`, when the +request would delete or replace anything: + +- `proxmox.apply`: the saved plan has any `delete` or `replace` action + (including `create_before_destroy`). The engine computes this from + `tofu show -json` of the plan. It is never taken from the request. +- `ansible.apply`: the play is not tagged `non_destructive` in the + inventory, or a `--check --diff` run reports a removed file, package, user + or service. +- `komodo.deploy`: the new compose removes a service or a named volume that + the running stack has. +- `forgejo.merge` with `delete_branch: true` is not destructive (the branch is + merged), but a merge into a protected branch other than the default is + `person_only`. +- `tailscale.*`: always `person_only`. + +The core treats the computed classification as binding. An approval through +a delegation is refused for a `person_only` or `destructive` request. If the +execute-time classification is higher than the one approved, for example +because the plan changed, the engine refuses with `risk_raised`, and the +worker must request again. + +### 1.6 Refused requests and splitting + +A worker that is refused a whole action must not get it by asking for parts +of it. A WI-2333 worker did that, splitting a refused action into smaller +steps. Four rules close the workaround: + +- **Recorded by item and target.** Denied, failed, rolled-back and withdrawn + requests stay in `action_requests` with their `work_item` and `target`. + The engine also reports classifier refusals it can see (a + `session_report_blocked` that names an action type) as rows in state + `refused`. +- **A re-request goes to a person.** A new request on the same + `(work_item, target)` within 24 h of a denied, failed, rolled-back or + refused one can't be approved through a delegation. The Inbox shows the + person the earlier requests and their outcomes beside the new one. Targets + match by prefix for narrower requests: `komodo:vogt-dev/service:a` matches a + refusal on `komodo:vogt-dev`. The window counts every session on the item, + so a second worker doesn't reset it. +- **`supersedes` makes a re-request explicit.** A worker re-requesting after a + fix names the earlier request. That doesn't make it delegable, but the + approver can see it is a deliberate retry, not a quiet split. +- **The classifier clause** (§5) forbids decomposing a refused or denied + action into narrower requests, component calls or sub-steps. + +*Test:* a worker denied `komodo.deploy stack=vogt-dev`, which then requests +`komodo.deploy stack=vogt-dev services=[a]` and `services=[b]`, gets both +requests routed to a person, with the denial shown. Neither is approvable by +its overseer's delegation. The same holds when the narrower requests come +from a second session on the same item. + +## 2. CLI and API surface + +### 2.1 Session side: `vogt-agent-auth` + +```text +vogt-agent-auth actions # list types, schemas, availability +vogt-agent-auth request --item WI-n \ + [--param key=value ...] [--params-file f.json] \ + --reason "…" # prints act_… and the params digest +vogt-agent-auth request-status # state and outcome +vogt-agent-auth request-withdraw # requester only, while pending +``` + +`request` calls `POST /api/agent-auth/action-requests` on loopback +(`VOGT_ENGINE_BROKER_URL`) with the session's broker token, which is the same +token `fetch` and `store` use. The engine validates the parameters against the +type's schema, adds the session's identity from the token, and relays the +request to the core with its own identity. The worker then calls +`session_report_blocked` naming the `act_…` id and stops. The engine wakes it +when the request reaches a final state (§3.4). + +`request` never takes a credential, never prints one, and never runs the +action. It's safe to allow in the classifier (§5). + +### 2.2 Core operations (registry → CLI, REST, MCP) + +| Operation | Scope | Who | +| --- | --- | --- | +| `action.request` | `work.write` | Only through the engine relay. A direct agent call is refused (`relay_required`), so the requester session is always the engine's word. A person may also request from the GUI or CLI. | +| `action.decide` (`approve` \| `deny`, `params_digest`, `reason`) | `work.write` | A person, or an oversight session within a delegation (§1.4). Every other agent principal is refused with `403 person_required`. | +| `action.withdraw` | `work.write` | The requester, or a person. | +| `action.list` / `action.get` | `read` | Filters: `state`, `work_item`, `action`, `requester`. | +| `action.delegation_set` / `_revoke` / `_list` | `admin` | A person only. | + +`action.decide` must carry the `params_digest` the approver saw. If it doesn't +match the row, the call is refused (`params_changed`). + +### 2.3 Engine routes + +| Route | Caller | Purpose | +| --- | --- | --- | +| `GET /api/actions/types` | any authenticated caller | The registry (§1.1). | +| `POST /api/agent-auth/action-requests` | a session broker token | Validate and relay a request (§2.1). Rate-limited per session: 10 an hour. | +| `GET /api/agent-auth/action-requests/{id}` | a session broker token | The requester's own requests only. | +| `POST /api/actions/{id}/execute` `{action, params, params_digest, approver, approval_kind}` | **the stack secret only** (`vogt-core`) | Run the executor. The engine recomputes the digest from the body and refuses a mismatch. | +| `POST /api/actions/{id}/confirm` | the stack secret | Disarm a dead-man (§3.3). | + +The engine half of "it was approved" is the same as for grants. Only the +core's stack-secret identity can make the engine execute, and the core sends +an execute request only after recording an approval. + +## 3. Executor contract + +Every executor runs in this order. Each step writes a `vogt::audit` line +(`event=action`, with `phase=`). + +### 3.1 Preconditions + +These are read-only checks, run with the engine credential at execute time, +not at request time. Things change between the request and the approval, so +the checks must run against the state at execution. Each type lists its own +checks (§4). Every type also checks two things: the work item is still open, +and the request is still `executing`. If any check fails, the request ends +`failed`, with the failing check named, and the target is untouched. + +### 3.2 Verify + +After `execute`, the engine reads the target back and checks that the +intended state now holds: a merge commit is on the base branch, a container is +healthy at the new image, an ACL ETag matches the applied policy. If +verification fails, the engine runs `rollback`. + +### 3.3 Rollback and the dead-man + +`execute` is preceded by a **before snapshot**, stored with the outcome by +digest and kept by the engine for rollback. A type's rollback restores that +snapshot. Where a rollback isn't possible (a merge is public history), the type +says so and its preconditions must carry more of the weight. + +A type with `deadman_seconds > 0` arms a timer once `verify` passes. If no +`confirm` arrives before the timer runs out, the engine rolls the change back +and the request ends `rolled_back`. The confirm comes from the core, on behalf +of a person or the requester's follow-up check. The dead-man exists for changes +that can cut off the path used to confirm them. The main case is a Tailscale +ACL that locks the tailnet out of the engine. The timer runs inside the engine +and doesn't need the network path the change might break. An engine restart +while the timer is armed rolls the change back at startup (fail closed). For +that, the snapshot is persisted under the engine's state directory, mode 0600. + +### 3.4 Audit and wake + +When the request reaches a final state, the engine returns the outcome to the +core. The core records it on the row and emits `action.completed`, carrying +the requester, approver, approval kind, action, params digest, item, outcome +and change id. It posts a `work_comment` on the item (kind `action`), and wakes +the requester with `session_input`, a single line: `ACTION act_… : `. +The engine audits the same facts on `vogt::audit`. Neither side ever records a +credential. + +### 3.5 Engine-held credentials + +The engine resolves an action's credential from the stack's agent-auth helper +(`vogt-agent-auth get VAR`) with a manifest entry flagged **`action`**, a new +flag beside `optional`, `ondemand` and `writable`. An entry flagged `action` is +**never** exported to a session, and it is never fetchable through +`/api/agent-auth/fetch` or a grant. The engine's broker refuses it the way it +refuses an undeclared name. The executor resolves the credential when it runs, +holds it in memory for that one call, and drops it afterwards. + +Suggested manifest lines (the ops overlay owns the real ones): + +```text +ACTION_FORGEJO_TOKEN FORGEJO_ACTIONS_BOT_TOKEN action +ACTION_KOMODO_KEY KOMODO_ACTIONS_API_KEY action +ACTION_KOMODO_SECRET KOMODO_ACTIONS_API_SECRET action +ACTION_TAILSCALE_OAUTH TAILSCALE_ACL_OAUTH_CLIENT action +``` + +Oversight creates these in phase 2 and stores them in Infisical (decided 2026-10-24). Each is a **dedicated bot credential** with only the rights its executor needs: +a Forgejo bot user with write on the named repos, a Komodo API key limited to +deploy on non-prod stacks where Komodo allows it, and a Tailscale OAuth client +with the `policy_file` and `devices` scopes. None is the operator's personal +token. + +## 4. The first action types + +### 4.1 `forgejo.merge` + +| | | +| --- | --- | +| Params | `repo` (`owner/name`, matched against `ENGINE_ACTION_FORGEJO_REPOS`), `pr` (integer), `head_sha` (40 hex), `style` (`rebase` \| `squash` \| `merge`, default `rebase`), `delete_branch` (bool, default `false`) | +| Risk | `standard` | +| Preconditions | The PR is open. Its head is exactly `head_sha`, so a push after approval voids it. It is mergeable, with no conflict. Every required status check on the base branch's protection is `success` at `head_sha`. The base branch is the repo default or one named in the protection rule. | +| Execute | `POST /repos/{repo}/pulls/{pr}/merge` with `Do = style` and `head_commit_id = head_sha`. **Never `force_merge`**: the executor has no code path that sets it. | +| Verify | The PR reports `merged`, and the base branch now contains `head_sha`'s tree (for `rebase`, the base tip's tree equals the merged result's). The merge commit SHA is recorded. | +| Rollback | `none`. A merge is public history. A follow-up revert is a separate, newly requested change. This is why the precondition pins `head_sha`. | +| Dead-man | `0` | + +This replaces oversight's personal-token merges in `indexarr/ops` (rollout +phase 3). Merging `indexarr/ops` main is not a deploy, so this type stays +`standard`. + +### 4.2 `komodo.deploy` + +| | | +| --- | --- | +| Params | `stack` (a Komodo stack name), `expected_commit` (optional: the ops repo SHA the stack should be at after `PullStack`), `services` (optional list) | +| Risk | `standard`, delegable, for non-prod stacks. `destructive` when the deploy removes a service or a named volume (§1.5). **Prod stacks are refused outright before phase 4**: the executor has no prod path until then. In phase 4 a prod deploy would be `person_only` and also need `ENGINE_ACTION_KOMODO_ALLOW_PROD=1`. | +| Preconditions | The stack exists and is not mid-deploy. If `expected_commit` is set, the stack's repo resolves to it after a refresh. The stack's `env_diff` against the last successful deploy is recorded in the outcome as names and fingerprints only, never values. | +| Before snapshot | The current image digests per service and the last successful deploy's update id. | +| Execute | `DeployStack` (with `services` if given). | +| Verify | Every service is running and healthy within `verify_timeout` (default 300 s), at a digest that is not the before digest unless nothing changed. | +| Rollback | Redeploy the before snapshot's digests (pinned), then verify again. A failed rollback ends `failed`, raises an Inbox entry of severity `high`, and leaves the stack as found. | +| Dead-man | `0` by default. `verify` covers health. A stack may set a dead-man in its delegation constraint where an external smoke test must confirm. | + +### 4.3 `proxmox.apply` + +This is the OpenTofu apply that ops#98 (WI-2334) calls `tofu.apply +root=proxmox`. The two names are the same type, and ops#98 should adopt +`proxmox.apply`. + +| | | +| --- | --- | +| Params | `root` (an `infra/` root, default `proxmox`), `commit` (the ops `main` SHA the plan was made at), `plan_sha256` (the digest of the encrypted saved plan the pipeline uploaded) | +| Risk | `standard`, delegable, when the plan has no delete or replace. `destructive` and `person_only` otherwise (§1.5). | +| Preconditions | `main` of the ops repo is still at `commit`. The engine fetches the saved plan, checks its SHA-256 is `plan_sha256`, and decrypts it with the state key. `tofu show -json` lists no `delete` and no `replace`, unless a person approved. The plan's `prior_state` serial matches the live state's serial, so nobody applied since. | +| Before snapshot | The state serial, and a copy of each touched VM's `qm config` (`/etc/pve/qemu-server/.conf`). | +| Execute | `tofu apply plan.tfplan`, the saved plan only. A plan the engine made itself is never applied. | +| Verify | A fresh `tofu plan` at `commit` is empty. | +| Rollback | Restore each touched VM's saved `qm config`. If that fails, raise an Inbox entry of severity `high`. A create is rolled back by a person (deleting is destructive). | +| Dead-man | `0` | + +### 4.4 `ansible.apply` + +| | | +| --- | --- | +| Params | `play` (a role or playbook in `infra/hosts`), `hosts` (an inventory group or host), `commit` (the ops SHA), `check_sha256` (the digest of the `--check --diff` output the request was made from) | +| Risk | `person_only` in phase 1. A play tagged `non_destructive` in the inventory can be made delegable by a person's decision. Destructive as in §1.5. | +| Preconditions | `main` is at `commit`. A fresh `--check --diff` on the same hosts has digest `check_sha256`, so the change is the one reviewed. `--limit` resolves to the same host set. | +| Before snapshot | The role's own backups (`.graft-bak` files, per ops#98 phase 3). | +| Execute | `ansible-playbook --limit ` at `commit`. | +| Verify | A second `--check` run reports zero changes (idempotent). | +| Rollback | The role's revert handlers, then a `--check` run again. | +| Dead-man | 180 s for plays that touch `sshd`, the firewall or Tailscale, with Tailscale SSH as the out-of-band path. `0` otherwise. | + +### 4.5 `tailscale.acl.apply` + +| | | +| --- | --- | +| Params | `policy_sha256` (the digest of the exact HuJSON to apply), `source` (`{repo, path, commit}` in the ops repo, which the engine reads itself), `expected_etag` (the live policy's ETag when the request was made) | +| Risk | `person_only`, always | +| Preconditions | The engine fetches `source` at `commit` and checks that its SHA-256 is `policy_sha256`. The live policy's ETag is still `expected_etag`, so nobody changed it since. `POST /api/v2/tailnet/-/acl/validate` accepts it. The policy's own `tests` block passes. A built-in **lockout check** confirms the engine's own node can still reach the Vogt front door and the Komodo core under the new policy, through a `tests` entry the executor adds if it is missing. | +| Before snapshot | The live policy body and ETag. | +| Execute | `POST /api/v2/tailnet/-/acl` with `If-Match: expected_etag`. | +| Verify | The live ETag changed, and the body's digest equals `policy_sha256`. | +| Rollback | Re-post the before body with `If-Match` on the new ETag. | +| Dead-man | **180 s**, the same as ops#98's `bin/deadman-revert.sh`. The person who approved, or the requester's check over the tailnet, must `confirm` within three minutes. Otherwise the engine restores the old policy. This is the guard against the change cutting off the path the confirmation would travel. | + +The Inbox entry for an ACL apply shows the **diff** between the live policy +and the proposed one, rendered by the engine from the two bodies. A digest +alone isn't reviewable. + +### 4.6 `tailscale.device.tag` + +| | | +| --- | --- | +| Params | `device` (stable node id, not hostname), `add` (tags), `remove` (tags), `expected_tags` (current set) | +| Risk | `person_only`, always | +| Preconditions | The device exists, and its current tags equal `expected_tags`. Every tag in `add` is defined in `tagOwners`. The change doesn't remove the last tag from a device that has no user owner, which would orphan the node. | +| Execute | `POST /api/v2/device/{id}/tags` with the resulting set. | +| Verify | The device's tags read back as the resulting set. | +| Rollback | Re-post `expected_tags`. | +| Dead-man | 180 s, as for `acl.apply`, because a tag change moves the node across ACL rules. | + +### 4.7 Later types + +`github.merge` follows `forgejo.merge`, with a GitHub App installation token +and the repository's required checks or merge queue. `infisical.set` overlaps +with the existing `writable` store broker. It is worth adding only for secrets +whose *value* the engine generates itself (rotation), so no session ever sees +it. + +## 5. The classifier allow rule + +The rule goes in `engine/deploy/driven-session-settings.json` under +`autoMode.allow`, beside "Approved Vogt Grant". It also adds `Bash(vogt-agent-auth request *)`, +`Bash(vogt-agent-auth request-status *)` and `Bash(vogt-agent-auth actions)` to +`permissions.allow`. + +> **Approved Action Request**: running `vogt-agent-auth request --item WI- …`, +> `vogt-agent-auth request-status `, `vogt-agent-auth request-withdraw ` +> or `vogt-agent-auth actions`. These commands ask the Vogt engine to carry out +> a named infrastructure action after a person, or a delegation a person set, +> approves it. They never perform the action, hold a credential, or change +> anything outside Vogt's request record. The engine runs the action only after +> that approval, with its own credential. This clears Modify Shared Resources, +> Production Changes and Merge Without Review for **making the request** +> alone. It never covers performing the action by any other means (Komodo, +> Tailscale, Forgejo or GitHub API calls, `curl`, the `komodo`, `tailscale` or +> `gh` CLIs), retrying an action the engine refused or failed by hand, +> fetching or using an action credential, or treating a request's approval as +> permission for anything other than the engine executing it. A denied, +> failed or rolled-back request is reported with `session_report_blocked`, +> not worked around. **A refused or denied action must never be split** into +> narrower requests, component calls or sub-steps that together achieve it +> (for example, deploying a denied stack service by service, or applying a +> denied plan resource by resource). After a refusal, the only routes are a +> new request for the whole action that names the refused one in +> `--supersedes`, or `session_report_blocked`. + +The protected-scope defaults stay as they are for any action done by hand. + +**Oversight sessions keep bypass**, as the operator decided. This design +doesn't change their posture. Oversight *should* use the executors for an +infrastructure change where one exists, so the change carries a request, an +approval and an outcome record. A Vogt charter can say so. Nothing enforces +it. + +## 6. Security analysis + +### 6.1 Request spoofing + +*Threat:* a request claims to come from a different session or item. +*Mitigation:* the core accepts `action.request` only through the engine relay +(`relay_required`). The relay takes the requester from the broker token, which +the engine minted for that one session and revokes when the session is +forgotten. It is never a body field. A person's request comes from the GUI +under their own principal. The work item must exist and be open. *Test:* a +session token calling `action.request` directly is refused, and a body +`requester` field is rejected by the schema. + +### 6.2 Parameter binding + +*Threat:* a person approves one thing and something else runs. This covers +parameters edited after approval, a PR head moved, a different policy file, +or a reason that misdescribes the change. +*Mitigation:* the parameters are canonical JSON, validated and frozen at +request time. The approval carries the `params_digest` the approver saw, and +the core refuses a mismatch. The engine recomputes the digest from the execute +body and refuses a mismatch again. Every mutable target is pinned by content +inside the parameters: `head_sha` for merges, `policy_sha256` and +`expected_etag` for ACLs, `expected_tags` for devices, optionally +`expected_commit` for deploys. The preconditions re-check those pins at +execute time. The Inbox renders the parameters (and, for ACLs, the diff), not +the reason. The reason is shown as untrusted text. + +### 6.3 Replay + +*Threat:* one approval is used twice, or an old approval is used late. +*Mitigation:* a request is single-use. `approved → executing` is a conditional +update, and the engine keeps the ids of executed requests and refuses a second +execute for the same `act_…` id (`409 already_executed`). An approval expires +if not executed within 15 minutes. Pins like `expected_etag` and `head_sha` +make a stale replay fail its preconditions even if everything else were +bypassed. + +### 6.4 Approver identity + +*Threat:* an agent approves its own request, or launders an approval through +another agent. +*Mitigation:* `action.decide` refuses every agent principal unless it is an +oversight session acting within an active delegation that a person set +(§1.4). The oversight role can only be assigned by a person (WI-957, +`role_refused`). Self-approval is refused. `person_only` types and prod targets +can't be delegated. Delegations expire, are rate-capped, and are listed with +their use counts. Every approval records `approval_kind`, so a reader can tell +a person's approval from a delegated one. + +### 6.5 The engine as executor + +*Threat:* something other than the core makes the engine execute. +*Mitigation:* the execute and confirm routes accept only the stack secret, +checked by the credential compared, as for grants. Action credentials carry +the `action` manifest flag and are never exported or fetchable. +**The same caveat as grants applies (oversight-grants invariants 2 and 10):** +sessions run as the engine's uid, and this deployment runs `identity` mode. +A session that reads `/proc/1/environ` or the core token file holds the stack +secret. In identity mode a session can read the bot credentials from the +vault directly. Approved actions therefore give the operator a **governed, +audited path the classifier can clear**, not a hard boundary, until sessions +run as a separate uid (WI-982) and the deployment runs brokered. The design +doesn't change when that lands. + +### 6.6 Executor faults + +*Threat:* an executor bug or a half-applied change. +*Mitigation:* the before snapshot, verify, rollback, the dead-man for +lockout-capable changes, and an Inbox entry of severity `high` on a failed +rollback. Each executor gets an integration test against a fake of its API, +including the refusal paths. `force_merge` and `--admin` have no code path. + +### 6.7 Denial of service through the Inbox + +*Threat:* a worker floods the Inbox with requests. +*Mitigation:* a per-session rate limit of 10 an hour at the relay, at most 3 +pending requests per session, and the parameters are schema-validated before +any row is written. These limits also cap fan-out: the pending count is +kept per `(work_item, target)` across every session on the item, as well as +per session, so splitting across workers doesn't buy more requests. + +## 7. Phased rollout (operator accepted 1–4) + +| Phase | Content | Layers | +| --- | --- | --- | +| **1. Engine** | The action-type registry and `GET /api/actions/types`. The request relay. The `action_requests` table and migration, with `target`, `risk`, `destructive` and `supersedes`. Refused-request recording and person routing (§1.6), with the splitting test. `action.request` / `_decide` (person only) / `_withdraw` / `_list` / `_get`. Inbox entry with parameters and Approve/Deny. Stack-secret `execute` and `confirm`. The `action` manifest flag. Executors for `forgejo.merge`, `komodo.deploy` (non-prod), `proxmox.apply`, `ansible.apply`, `tailscale.acl.apply` and `tailscale.device.tag`. `vogt-agent-auth actions\|request\|request-status\|request-withdraw`. Wake on completion. | core, engine, PWA, deploy | +| **2. Policy** | `action_delegations` and `action.delegation_*` (person only). Delegated approval by oversight. The "Approved Action Request" classifier rule and the `permissions.allow` lines in the driven-session settings. Oversight creates the bot credentials and stores them in Infisical. Ops overlay: the `action` manifest lines. | core, deploy, ops | +| **3. Retire personal-token merges** | Oversight's `indexarr/ops` merges move to `forgejo.merge` under a delegation for `indexarr/ops`, `style: rebase`. The operator's personal Forgejo token is removed from oversight's reach. | ops, process | +| **4. Expand** | `github.merge`, `infisical.set` (engine-generated values only), the prod `komodo.deploy` opt-in (not before this phase), and further types on demand. Each new type is a code change with its own review. | engine | + +The classifier rule ships in phase 2, not phase 1. Phase 1 is exercised by +people and oversight sessions, which have bypass, so the executors are proven +on dev before driven workers can reach them. + +## 8. Decisions + +Decided by the operator on 2026-10-24: + +1. **Oversight approval.** Oversight may approve requests from workers it + started, within a standing delegation for the non-destructive types + `forgejo.merge`, `komodo.deploy` on non-prod stacks and `proxmox.apply` + with no deletes. `tailscale.*`, prod and anything destructive stay + person-only (§1.4, §1.5). +2. **Bot credentials.** Oversight creates the phase-2 credentials (a Forgejo + bot user, a scoped Komodo API key, a Tailscale OAuth client) and stores + them in Infisical (§3.5). +3. **Prod `komodo.deploy`.** Not allowed before phase 4. Whether it is + allowed at all is decided then (§4.2). + +Still open: + +4. **Identity mode.** As with grants (oversight-grants question 6), the + guarantee here is real only in brokered mode with a separate session uid. + Does adopting approved actions change the case for leaving identity mode? +5. **Delegable Ansible plays.** Should a play tagged `non_destructive` become + delegable (§4.4), or does `ansible.apply` stay person-only? diff --git a/docs/design/composer-attachments.md b/docs/design/composer-attachments.md new file mode 100644 index 000000000..2e3fdf5f1 --- /dev/null +++ b/docs/design/composer-attachments.md @@ -0,0 +1,282 @@ +# Composer attachments (WI-1829) + +Status: design, no implementation. 2026-10-18. + +## Problem + +To hand an agent a screenshot or file from the phone today you open the Files +panel, upload into some folder, then type the path into the composer by hand. +The goal is one gesture from any composer (chat, assistant, terminal session) +or from the Android share sheet. That gesture puts the file somewhere the agent +can read it and inserts the reference for you. + +## What exists + +| Piece | Where | Reused how | +|---|---|---| +| Streaming upload `PUT /api/files/upload?path=&create_parents=` | `engine/server/src/files.rs` `upload_file` | The write path stays as it is: spool to a temp file in the target dir, sha256, atomic rename. Cap is `MAX_UPLOAD_BYTES` = 10 GiB. | +| Capability gate | `engine/server/src/auth.rs` (`FilesystemWrite` on the upload route, WI-1020) | Attachments need the same capability. No new grant. | +| Client `api.uploadFile(path, blob, create_parents)` | `web/src/api.ts` | Called as is by the new composer helper. | +| Chat composer (`textarea.chat-composer`) | `web/src/Chat.tsx` | Gets the attach button, paste and drop. | +| Assistant composer | `web/src/Assistant.tsx` | Same. | +| Terminal composer / ModKeyRow, xterm paste | `web/src/Terminal.tsx`, `ModKeyRow.tsx`, `terminalInput.ts` | Same, plus a paste-image hook on xterm. | +| Chat turn | `engine/server/src/chat_codex.rs` `turn_start` sends `input: [{type:"text"}]` to the Codex app-server | Gains `{type:"localImage", path}` items. | +| Session input `POST /api/sessions/{id}/input {text, submit}` | `engine/server/src/api.rs` | No change. The path goes in as text. | +| Worktree cleanup | `engine/server/src/worktree_clean.rs` (WI-1304/WI-1750) | Extended to remove the attachments dir when the session stops. | +| Android wrapper | `mobile/android` (Capacitor 8). `ServerActivity` is the exported launcher root and the "Connect to Vogt" entry, deliberately **not** `singleTask` (WI-924). It starts `MainActivity` (not exported), which holds the signed-in WebView. | Gets a new exported `ShareActivity` that receives `SEND` / `SEND_MULTIPLE` and hands off to `MainActivity`. `ServerActivity` and its launch mode are unchanged. | + +## Recommendation + +### 1. One engine route: `POST /api/attachments` + +This is a thin wrapper over the upload code, not a new write path. The client +does not pick the path. The engine does, so a composer can never write into the +repo or anywhere else in the tree. + +``` +POST /api/attachments?target=session:|chat:|assistant&name= +Content-Type: (raw body, streamed like /api/files/upload) +→ 201 { id, path, rel_path, name, mime, bytes, sha256, width?, height?, stripped_exif: bool } +GET /api/attachments/{id}/thumb (≤256px JPEG/WebP, for the chip and transcript) +DELETE /api/attachments/{id} (remove a chip before sending) +``` + +- **Auth:** it takes `FilesystemWrite`, the same as `/api/files/upload`. The + caller must also be able to see the target: the same `may_show` policy that + gates session and chat reads (WI-1005/WI-1020). An agent token that can write + files can attach, as it could upload already. That is not a new power. +- **Where the file lands:** in the target's working directory, at + `/.vogt/attachments/-.`. + - The session cwd is the linked worktree for worker sessions and the chosen + cwd otherwise. A chat uses its own cwd, and the assistant uses + `/assistant/attachments/`. + - It is never written at the repo root. `.vogt/` is git-ignored in two ways: + the engine writes `.vogt/.gitignore` containing `*` when it creates the + dir, so it never shows up in `git status` even for repos that don't list + it. + - The file is in the agent's own sandbox root, so every CLI can read it + without an extra permission grant. +- **Names:** the engine slugifies the name, rejects separators and dotfiles, + and appends `-2`, `-3`, … on collision. Original names are kept only in the + response. +- **Limits:** + - Per-file cap `ENGINE_ATTACHMENT_MAX_BYTES`, default **25 MiB**. A phone + photo is 3–8 MB, and anything larger belongs in the Files panel. + - Per-target cap of 200 MiB or 100 files. Over either cap the request gets + `413` with a message pointing at Files. + - A MIME allow-list is not enforced, because any file is useful. Only + images get thumbnails and image items. +- **Privacy:** for `image/jpeg`, `image/heic` and `image/png`, the engine + strips EXIF, GPS and XMP metadata before the rename. Rust crates + (`img-parts`) do this without re-encoding. It also applies the orientation + tag to the pixels, or keeps only Orientation, so photos don't come out + sideways. `stripped_exif` reports what happened. Stripping happens on the + server so that every entry point (picker, share sheet, paste) gets it. A + client-side strip would be a second layer, not a replacement. +- **Retention:** the attachments dir is removed when the session is + **stopped or removed**. It is not removed on hibernation, because a woken + session still needs its references. This uses the same hook and the same + "linked worktree only, never the main checkout" guards as WI-1750. A chat's + dir goes when the chat is archived. Assistant attachments expire after 7 + days by a sweep. A cwd that is a main checkout (a non-worktree session) + still has its `.vogt/attachments` cleared on stop, because the dir is + engine-owned and ignored. +- **Audit:** `vogt::audit event=attachment.create` with the target, bytes, + mime and principal. The audit never records content or the full original + name. + +### 2. What gets inserted + +The inserted text is the **absolute path**, with the **thumbnail chip** shown in +the PWA only. Markdown image syntax is not used: a CLI reads it as plain text, +and it costs tokens for nothing. + +| Agent | How it best takes an image | What Vogt sends | +|---|---|---| +| Claude Code (TTY) | It reads an image when the prompt contains a local path (`Read` tool, multimodal). | Text: ` /abs/path.png ` | +| Codex CLI (TTY) | `codex -i` at launch only. In the TUI, a pasted path is read by its file tools. Images are viewed with its `view_image` tool. | Text path (same as above) | +| Chat (Codex app-server) | The `turn/start` input accepts `{type:"localImage", path}` as a real image item. | Text item plus a `localImage` item per image | +| opencode | `@file` references, and image paths in the prompt are attached. | Text: `@/abs/path.png` (the opencode adapter rewrites a bare path) | +| Klaudia | Uses the opencode backend. Same as opencode, to be confirmed in item 6 below. | Same as opencode | +| Assistant (OpenAI-compatible) | `image_url` content parts with a data URI, only when the configured model supports vision. | Path in the text and, if vision is on, a data-URI part. Otherwise the text names the attachment and its path so tools can fetch it. | + +The composer keeps the chips separate from the draft text. On send, it appends +one path per line at the end of the text, unless the user has placed a +reference token inline. An agent-CLI adapter in `agent_cli.rs` owns the +per-CLI format (bare path vs `@path`), so the PWA does not hard-code CLI +behaviour. + +### 3. Terminal sessions: type into the PTY, no side channel + +The path is written through the existing input path: the WS binary frame, or +`session_input` for MCP and driving. It is sent as a **bracketed paste** +(`ESC[200~ … ESC[201~`), so the TUIs treat it as pasted text and don't act on +each character. Bracketed paste is also how Claude Code and Codex detect pasted +image paths. **`submit` is never set**: the user still presses Enter, so an +attachment can't fire a half-written prompt. + +A side channel (an engine-held "pending attachments" list that each CLI polls) +was rejected. None of the CLIs have an input for it, and it would split the +prompt across two transports. + +`session_input` needs no change. Agents that want to attach for another session +call `POST /api/attachments` and then `session_input` with the path. A +`session_attach` MCP tool can wrap both later; that is optional (item 7). + +### 4. UI + +The same three entry points on every composer: + +``` +┌ chat / assistant / terminal composer ───────────────────────────┐ +│ [🖼 screenshot-0912.png ×] [📄 trace.log ×] ← chips, upload % │ +│ ┌──────────────────────────────────────────────────┐ ┌──┐ ┌──┐ │ +│ │ why does this crash? │ │📎│ │➤ │ │ +│ └──────────────────────────────────────────────────┘ └──┘ └──┘ │ +└─────────────────────────────────────────────────────────────────┘ +📎 → sheet: [ Photo library ] [ Camera ] [ File ] +``` + +- **📎 button:** a hidden ``, with + `accept="image/*"` + `capture="environment"` for "Camera". In the Android + WebView, Capacitor's file chooser already returns content URIs for + ``, so no plugin is needed. +- **Chips:** + - Each chip uploads on pick and shows progress. A thumbnail comes from a + local `URL.createObjectURL` until `/thumb` is ready. + - × calls `DELETE`. Send is disabled while any chip is uploading. + - On a terminal pane, the chip row is the ModKeyRow tray. Tapping a chip + inserts the path immediately (see §3), because a PTY has no "draft". +- **Desktop paste:** an `onPaste` handler on each textarea, and an xterm + `paste` listener on terminals. When `clipboardData.files` holds an image, + it is attached as `paste-